From 0ad084d37dc59fb6ccdc2c9f3a13662051ef2f21 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Thu, 18 Apr 2024 12:54:27 +0530 Subject: [PATCH 01/37] Add 1.3.16 release notes (#1156) (#1158) * Add 1.3.16 release notes Signed-off-by: Shreyansh Ray * Fix review comments Signed-off-by: Shreyansh Ray --------- Signed-off-by: Shreyansh Ray (cherry picked from commit df1e137c0561cb5eb06cda76298ea8d65f9d0199) Co-authored-by: rayshrey <121871912+rayshrey@users.noreply.github.com> --- ...ensearch-index-management.release-notes-1.3.16.0.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 release-notes/opensearch-index-management.release-notes-1.3.16.0.md diff --git a/release-notes/opensearch-index-management.release-notes-1.3.16.0.md b/release-notes/opensearch-index-management.release-notes-1.3.16.0.md new file mode 100644 index 000000000..649a6f1aa --- /dev/null +++ b/release-notes/opensearch-index-management.release-notes-1.3.16.0.md @@ -0,0 +1,10 @@ +## Version 1.3.16.0 2024-4-18 + +Compatible with OpenSearch 1.3.16 + +### Maintenance +* Increment version to 1.3.16 ([#1131](https://github.com/opensearch-project/index-management/pull/1131)) +* Force resolve logback deps to mitigate CVE-2023-6378 ([#1125](https://github.com/opensearch-project/index-management/pull/1125)) + +### Documentation +* Version 1.3.16 release notes ([#1156](https://github.com/opensearch-project/index-management/pull/1156)) From ffabd8692767fb39cb3fd5bd3e4d7d74b62d51af Mon Sep 17 00:00:00 2001 From: Oleg Kravchuk Date: Thu, 2 May 2024 21:49:34 +0300 Subject: [PATCH 02/37] start_time npe fix for Rollup and Transform jobs (#1164) --- .../org/opensearch/indexmanagement/rollup/model/Rollup.kt | 3 ++- .../opensearch/indexmanagement/transform/model/Transform.kt | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/org/opensearch/indexmanagement/rollup/model/Rollup.kt b/src/main/kotlin/org/opensearch/indexmanagement/rollup/model/Rollup.kt index b15145040..19da98e8b 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/rollup/model/Rollup.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/rollup/model/Rollup.kt @@ -340,9 +340,10 @@ data class Rollup( // TODO: Make startTime public in Job Scheduler so we can just directly check the value if (seqNo == SequenceNumbers.UNASSIGNED_SEQ_NO || primaryTerm == SequenceNumbers.UNASSIGNED_PRIMARY_TERM) { if (schedule is IntervalSchedule) { - schedule = IntervalSchedule(schedule.startTime, schedule.interval, schedule.unit, schedule.delay ?: 0) + schedule = IntervalSchedule(schedule.startTime ?: Instant.now(), schedule.interval, schedule.unit, schedule.delay ?: 0) } } + return Rollup( id = id, seqNo = seqNo, diff --git a/src/main/kotlin/org/opensearch/indexmanagement/transform/model/Transform.kt b/src/main/kotlin/org/opensearch/indexmanagement/transform/model/Transform.kt index 14dd49ac0..fb017d21f 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/transform/model/Transform.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/transform/model/Transform.kt @@ -397,7 +397,7 @@ data class Transform( if (seqNo == SequenceNumbers.UNASSIGNED_SEQ_NO || primaryTerm == SequenceNumbers.UNASSIGNED_PRIMARY_TERM) { // we instantiate the start time if (schedule is IntervalSchedule) { - schedule = IntervalSchedule(schedule.startTime, schedule.interval, schedule.unit) + schedule = IntervalSchedule(schedule.startTime ?: Instant.now(), schedule.interval, schedule.unit) } // we clear out metadata if its a new job From 2101fd1c0ad920574e5c17833442ea22bb4d6a51 Mon Sep 17 00:00:00 2001 From: Sooraj Sinha <81695996+soosinha@users.noreply.github.com> Date: Fri, 3 May 2024 07:55:03 +0530 Subject: [PATCH 03/37] Add release notes for 2.14 (#1165) --- .../opensearch-index-management.release-notes-2.14.0.0.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 release-notes/opensearch-index-management.release-notes-2.14.0.0.md diff --git a/release-notes/opensearch-index-management.release-notes-2.14.0.0.md b/release-notes/opensearch-index-management.release-notes-2.14.0.0.md new file mode 100644 index 000000000..dc53f16b2 --- /dev/null +++ b/release-notes/opensearch-index-management.release-notes-2.14.0.0.md @@ -0,0 +1,3 @@ +## Version 2.14.0.0 2024-05-02 + +Compatible with OpenSearch 2.14.0 From acbc9306d73134c706eb0ca014bcee417f3cad5c Mon Sep 17 00:00:00 2001 From: Varun Lodaya Date: Mon, 13 May 2024 11:24:50 +0530 Subject: [PATCH 04/37] Updating security reachout email (#1171) --- README.md | 2 +- SECURITY.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 23a014b31..c7e738c99 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ This project has adopted the [Amazon Open Source Code of Conduct](CODE_OF_CONDUC ## Security -If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public GitHub issue. +If you discover a potential security issue in this project we ask that you notify OpenSearch Security directly via email to security@opensearch.org. Please do **not** create a public GitHub issue. ## License diff --git a/SECURITY.md b/SECURITY.md index e0ecd9024..be4ac7463 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,3 +1,3 @@ ## Reporting a Vulnerability -If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/) or directly via email to aws-security@amazon.com. Please do **not** create a public GitHub issue. \ No newline at end of file +If you discover a potential security issue in this project we ask that you notify OpenSearch Security directly via email to security@opensearch.org. Please do **not** create a public GitHub issue. From d4ee795e22f4490b78662f171f62d566a81c1abc Mon Sep 17 00:00:00 2001 From: Harshita Kaushik <112249538+harshitakaushik-dev@users.noreply.github.com> Date: Thu, 30 May 2024 05:35:12 +0530 Subject: [PATCH 05/37] Step Metadata Update on Index Rollover Timeout (#1174) Co-authored-by: Harshita Kaushik --- .../indexstatemanagement/Step.kt | 1 + .../ManagedIndexRunner.kt | 17 ++++++++----- .../action/ActionTimeoutIT.kt | 25 +++++++++++++++---- 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/Step.kt b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/Step.kt index 16c41c8f1..8de821936 100644 --- a/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/Step.kt +++ b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/Step.kt @@ -56,6 +56,7 @@ abstract class Step(val name: String, val isSafeToDisableOn: Boolean = true) { CONDITION_NOT_MET("condition_not_met"), FAILED("failed"), COMPLETED("completed"), + TIMED_OUT("timed_out"), ; override fun toString(): String { diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/ManagedIndexRunner.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/ManagedIndexRunner.kt index ab7ed58d4..ea7ed3765 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/ManagedIndexRunner.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/ManagedIndexRunner.kt @@ -91,6 +91,7 @@ import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedInde import org.opensearch.indexmanagement.spi.indexstatemanagement.model.PolicyRetryInfoMetaData import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StateMetaData import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData import org.opensearch.jobscheduler.spi.JobExecutionContext import org.opensearch.jobscheduler.spi.LockModel import org.opensearch.jobscheduler.spi.ScheduledJobParameter @@ -330,14 +331,18 @@ object ManagedIndexRunner : if (action?.hasTimedOut(currentActionMetaData) == true) { val info = mapOf("message" to "Action timed out") logger.error("Action=${action.type} has timed out") - val updated = - updateManagedIndexMetaData( - managedIndexMetaData - .copy(actionMetaData = currentActionMetaData?.copy(failed = true), info = info), - ) + + val updatedIndexMetaData = managedIndexMetaData.copy( + actionMetaData = currentActionMetaData?.copy(failed = true), + stepMetaData = step?.let { StepMetaData(it.name, System.currentTimeMillis(), Step.StepStatus.TIMED_OUT) }, + info = info, + ) + + val updated = updateManagedIndexMetaData(updatedIndexMetaData) + if (updated.metadataSaved) { disableManagedIndexConfig(managedIndexConfig) - publishErrorNotification(policy, managedIndexMetaData) + publishErrorNotification(policy, updatedIndexMetaData) } return } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ActionTimeoutIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ActionTimeoutIT.kt index 8d26405a8..54bdd764c 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ActionTimeoutIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ActionTimeoutIT.kt @@ -8,8 +8,10 @@ package org.opensearch.indexmanagement.indexstatemanagement.action import org.opensearch.indexmanagement.indexstatemanagement.IndexStateManagementRestTestCase import org.opensearch.indexmanagement.indexstatemanagement.step.open.AttemptOpenStep import org.opensearch.indexmanagement.indexstatemanagement.step.rollover.AttemptRolloverStep +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionMetaData import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData import org.opensearch.indexmanagement.waitFor import java.time.Instant import java.util.Locale @@ -22,9 +24,9 @@ class ActionTimeoutIT : IndexStateManagementRestTestCase() { val policyID = "${testIndexName}_testPolicyName_1" val testPolicy = """ - {"policy":{"description":"Default policy","default_state":"rolloverstate","states":[ - {"name":"rolloverstate","actions":[{"timeout":"1s","rollover":{"min_doc_count":100}}], - "transitions":[]}]}} + {"policy":{"description":"Default policy","default_state":"rolloverstate","states":[ + {"name":"rolloverstate","actions":[{"timeout":"1s","rollover":{"min_doc_count":100}}], + "transitions":[]}]}} """.trimIndent() createPolicyJson(testPolicy, policyID) @@ -60,11 +62,24 @@ class ActionTimeoutIT : IndexStateManagementRestTestCase() { fun(actionMetaDataMap: Any?): Boolean = assertActionEquals( ActionMetaData( - name = RolloverAction.name, startTime = Instant.now().toEpochMilli(), index = 0, - failed = true, consumedRetries = 0, lastRetryTime = null, actionProperties = null, + name = RolloverAction.name, + startTime = Instant.now().toEpochMilli(), + index = 0, + failed = true, + consumedRetries = 0, + lastRetryTime = null, + actionProperties = null, ), actionMetaDataMap, ), + StepMetaData.STEP to + fun(stepMetaDataMap: Any?): Boolean = + assertStepEquals( + StepMetaData( + "attempt_rollover", Instant.now().toEpochMilli(), Step.StepStatus.TIMED_OUT, + ), + stepMetaDataMap, + ), ), ), getExplainMap(indexName), From a049fdb423efcb02fb08a10f3b984fe392d09bab Mon Sep 17 00:00:00 2001 From: bowenlan-amzn Date: Fri, 7 Jun 2024 16:29:31 -0700 Subject: [PATCH 06/37] Update rollover.http (#1182) --- worksheets/ism/rollover.http | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/worksheets/ism/rollover.http b/worksheets/ism/rollover.http index d181f387c..e74140361 100644 --- a/worksheets/ism/rollover.http +++ b/worksheets/ism/rollover.http @@ -28,7 +28,7 @@ Content-Type: application/json "error_notification": { "destination": { "chime": { - "url": "https://hooks.chime.aws/incomingwebhooks/90edf596-a8f0-4451-9003-97a44e6c6105?token=WXRzWm1VVWt8MXxwMzJjUENNWVpJTXZXTXg5NUdDYWdtaUY4ZHlvTzR6bW5rS0ZJLWxQX2dN" + "url": "<>" } }, "message_template": { From 52f331fde465f1dd79667fd7c5740e270e430dab Mon Sep 17 00:00:00 2001 From: Vikas Bansal <43470111+vikasvb90@users.noreply.github.com> Date: Fri, 14 Jun 2024 23:45:02 +0530 Subject: [PATCH 07/37] Add release notes for 2.15 (#1190) Signed-off-by: vikasvb90 --- .../opensearch-index-management.release-notes-2.15.0.0.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 release-notes/opensearch-index-management.release-notes-2.15.0.0.md diff --git a/release-notes/opensearch-index-management.release-notes-2.15.0.0.md b/release-notes/opensearch-index-management.release-notes-2.15.0.0.md new file mode 100644 index 000000000..002435041 --- /dev/null +++ b/release-notes/opensearch-index-management.release-notes-2.15.0.0.md @@ -0,0 +1,7 @@ +## Version 2.15.0.0 2024-06-14 + +Compatible with OpenSearch 2.15.0 + +### Bug fixes + +* Step Metadata Update on Index Rollover Timeout ([#1174](https://github.com/opensearch-project/index-management/pull/1174)) \ No newline at end of file From c5425a52e6e7d5fd0f87bf770436527c8911ee08 Mon Sep 17 00:00:00 2001 From: Harshita Kaushik <112249538+harshitakaushik-dev@users.noreply.github.com> Date: Thu, 4 Jul 2024 16:04:27 +0530 Subject: [PATCH 08/37] Index Management Action Metrics (#1195) * Initial integration of TelemetryAwarePlugin to ISM Signed-off-by: harycash * Initial integration of TelemetryAwarePlugin to ISM Signed-off-by: harycash * Initial integration of TelemetryAwarePlugin to ISM with Rollover Step Integration Signed-off-by: harycash * Initial integration of TelemetryAwarePlugin to ISM with Rollover Step Integration Signed-off-by: harycash * Additional actions metrics with requested changes from previous commit Signed-off-by: harycash * Fixed Build Issues Signed-off-by: harycash * Fixed Build Issues Signed-off-by: harycash * Fixed Build Issues, Added new metric : Cumulative Latency Signed-off-by: harycash * Fixed Build Issues, Added new metric : Cumulative Latency Signed-off-by: harycash * Requested Changes and Addition of Metrics to all the remaining Actions Signed-off-by: harycash * Updates on Action Metrics Signed-off-by: harycash * Updates on Action Metrics Signed-off-by: harycash * Build issues fixed Signed-off-by: harycash * Build issues fixed Signed-off-by: harycash --------- Signed-off-by: harycash Co-authored-by: harycash --- .../indexstatemanagement/Step.kt | 100 ++++++++++++++- .../metrics/IndexManagementActionsMetrics.kt | 116 ++++++++++++++++++ .../actionmetrics/AliasActionMetrics.kt | 53 ++++++++ .../actionmetrics/AllocationActionMetrics.kt | 53 ++++++++ .../actionmetrics/CloseActionMetrics.kt | 53 ++++++++ .../actionmetrics/DeleteActionMetrics.kt | 53 ++++++++ .../actionmetrics/ForceMergeActionMetrics.kt | 53 ++++++++ .../actionmetrics/MoveShardActionMetrics.kt | 53 ++++++++ .../NotificationActionMetrics.kt | 53 ++++++++ .../actionmetrics/OpenActionMetrics.kt | 53 ++++++++ .../ReplicaCountActionMetrics.kt | 53 ++++++++ .../actionmetrics/RolloverActionMetrics.kt | 53 ++++++++ .../SetIndexPriorityActionMetrics.kt | 53 ++++++++ .../actionmetrics/SetReadOnlyActionMetrics.kt | 53 ++++++++ .../actionmetrics/ShrinkActionMetrics.kt | 53 ++++++++ .../actionmetrics/SnapshotActionMetrics.kt | 53 ++++++++ .../actionmetrics/TransitionActionMetrics.kt | 53 ++++++++ .../indexmanagement/IndexManagementPlugin.kt | 14 ++- .../ManagedIndexRunner.kt | 10 +- .../step/AttemptCloseStepTests.kt | 47 +++++-- .../step/AttemptDeleteStepTests.kt | 47 +++++-- .../step/AttemptOpenStepTests.kt | 39 +++++- .../step/AttemptRolloverStepTests.kt | 42 +++++-- .../step/AttemptSetIndexPriorityStepTests.kt | 38 ++++-- .../step/AttemptSetReplicaCountStepTests.kt | 52 ++++++-- .../step/AttemptSnapshotStepTests.kt | 36 +++++- .../step/AttemptTransitionStepTests.kt | 36 ++++-- .../step/SetReadOnlyStepTests.kt | 40 +++++- 28 files changed, 1339 insertions(+), 73 deletions(-) create mode 100644 spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/IndexManagementActionsMetrics.kt create mode 100644 spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/AliasActionMetrics.kt create mode 100644 spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/AllocationActionMetrics.kt create mode 100644 spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/CloseActionMetrics.kt create mode 100644 spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/DeleteActionMetrics.kt create mode 100644 spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/ForceMergeActionMetrics.kt create mode 100644 spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/MoveShardActionMetrics.kt create mode 100644 spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/NotificationActionMetrics.kt create mode 100644 spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/OpenActionMetrics.kt create mode 100644 spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/ReplicaCountActionMetrics.kt create mode 100644 spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/RolloverActionMetrics.kt create mode 100644 spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/SetIndexPriorityActionMetrics.kt create mode 100644 spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/SetReadOnlyActionMetrics.kt create mode 100644 spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/ShrinkActionMetrics.kt create mode 100644 spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/SnapshotActionMetrics.kt create mode 100644 spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/TransitionActionMetrics.kt diff --git a/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/Step.kt b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/Step.kt index 8de821936..b6fb65187 100644 --- a/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/Step.kt +++ b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/Step.kt @@ -9,6 +9,7 @@ import org.apache.logging.log4j.Logger import org.opensearch.core.common.io.stream.StreamInput import org.opensearch.core.common.io.stream.StreamOutput import org.opensearch.core.common.io.stream.Writeable +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.IndexManagementActionsMetrics import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData @@ -27,12 +28,109 @@ abstract class Step(val name: String, val isSafeToDisableOn: Boolean = true) { abstract suspend fun execute(): Step - fun postExecute(logger: Logger): Step { + fun postExecute( + logger: Logger, + indexManagementActionMetrics: IndexManagementActionsMetrics, + step: Step, + startingManagedIndexMetaData: ManagedIndexMetaData, + ): Step { logger.info("Finished executing $name for ${context?.metadata?.index}") + val updatedStepMetaData = step.getUpdatedManagedIndexMetadata(startingManagedIndexMetaData) + emitTelemetry(indexManagementActionMetrics, updatedStepMetaData, logger) this.context = null return this } + private fun emitTelemetry( + indexManagementActionMetrics: IndexManagementActionsMetrics, + updatedStepMetaData: ManagedIndexMetaData, + logger: Logger, + ) { + when (context?.metadata?.actionMetaData?.name) { + IndexManagementActionsMetrics.ROLLOVER -> indexManagementActionMetrics.getActionMetrics( + IndexManagementActionsMetrics.ROLLOVER, + ) + ?.emitMetrics(context!!, indexManagementActionMetrics, updatedStepMetaData.stepMetaData) + + IndexManagementActionsMetrics.FORCE_MERGE -> indexManagementActionMetrics.getActionMetrics( + IndexManagementActionsMetrics.FORCE_MERGE, + ) + ?.emitMetrics(context!!, indexManagementActionMetrics, updatedStepMetaData.stepMetaData) + + IndexManagementActionsMetrics.DELETE -> indexManagementActionMetrics.getActionMetrics( + IndexManagementActionsMetrics.DELETE, + ) + ?.emitMetrics(context!!, indexManagementActionMetrics, updatedStepMetaData.stepMetaData) + + IndexManagementActionsMetrics.REPLICA_COUNT -> indexManagementActionMetrics.getActionMetrics( + IndexManagementActionsMetrics.REPLICA_COUNT, + ) + ?.emitMetrics(context!!, indexManagementActionMetrics, updatedStepMetaData.stepMetaData) + + IndexManagementActionsMetrics.TRANSITION -> indexManagementActionMetrics.getActionMetrics( + IndexManagementActionsMetrics.TRANSITION, + ) + ?.emitMetrics(context!!, indexManagementActionMetrics, updatedStepMetaData.stepMetaData) + + IndexManagementActionsMetrics.NOTIFICATION -> indexManagementActionMetrics.getActionMetrics( + IndexManagementActionsMetrics.NOTIFICATION, + ) + ?.emitMetrics(context!!, indexManagementActionMetrics, updatedStepMetaData.stepMetaData) + + IndexManagementActionsMetrics.CLOSE -> indexManagementActionMetrics.getActionMetrics( + IndexManagementActionsMetrics.CLOSE, + ) + ?.emitMetrics(context!!, indexManagementActionMetrics, updatedStepMetaData.stepMetaData) + + IndexManagementActionsMetrics.SET_INDEX_PRIORITY -> indexManagementActionMetrics.getActionMetrics( + IndexManagementActionsMetrics.SET_INDEX_PRIORITY, // problem in test + ) + ?.emitMetrics(context!!, indexManagementActionMetrics, updatedStepMetaData.stepMetaData) + + IndexManagementActionsMetrics.OPEN -> indexManagementActionMetrics.getActionMetrics( + IndexManagementActionsMetrics.OPEN, + ) + ?.emitMetrics(context!!, indexManagementActionMetrics, updatedStepMetaData.stepMetaData) + + IndexManagementActionsMetrics.MOVE_SHARD -> indexManagementActionMetrics.getActionMetrics( + IndexManagementActionsMetrics.MOVE_SHARD, + ) + ?.emitMetrics(context!!, indexManagementActionMetrics, updatedStepMetaData.stepMetaData) + + IndexManagementActionsMetrics.SET_READ_ONLY -> indexManagementActionMetrics.getActionMetrics( + IndexManagementActionsMetrics.SET_READ_ONLY, + ) + ?.emitMetrics(context!!, indexManagementActionMetrics, updatedStepMetaData.stepMetaData) + + IndexManagementActionsMetrics.SHRINK -> indexManagementActionMetrics.getActionMetrics( + IndexManagementActionsMetrics.SHRINK, + ) + ?.emitMetrics(context!!, indexManagementActionMetrics, updatedStepMetaData.stepMetaData) + + IndexManagementActionsMetrics.SNAPSHOT -> indexManagementActionMetrics.getActionMetrics( + IndexManagementActionsMetrics.SNAPSHOT, + ) + ?.emitMetrics(context!!, indexManagementActionMetrics, updatedStepMetaData.stepMetaData) + + IndexManagementActionsMetrics.ALIAS_ACTION -> indexManagementActionMetrics.getActionMetrics( + IndexManagementActionsMetrics.ALIAS_ACTION, + ) + ?.emitMetrics(context!!, indexManagementActionMetrics, updatedStepMetaData.stepMetaData) + + IndexManagementActionsMetrics.ALLOCATION -> indexManagementActionMetrics.getActionMetrics( + IndexManagementActionsMetrics.ALLOCATION, + ) + ?.emitMetrics(context!!, indexManagementActionMetrics, updatedStepMetaData.stepMetaData) + + else -> { + logger.info( + "Action Metrics is not supported for this action [%s]", + context?.metadata?.actionMetaData?.name, + ) + } + } + } + abstract fun getUpdatedManagedIndexMetadata(currentMetadata: ManagedIndexMetaData): ManagedIndexMetaData abstract fun isIdempotent(): Boolean diff --git a/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/IndexManagementActionsMetrics.kt b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/IndexManagementActionsMetrics.kt new file mode 100644 index 000000000..725893a21 --- /dev/null +++ b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/IndexManagementActionsMetrics.kt @@ -0,0 +1,116 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.spi.indexstatemanagement.metrics + +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.actionmetrics.AliasActionMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.actionmetrics.AllocationActionMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.actionmetrics.CloseActionMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.actionmetrics.DeleteActionMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.actionmetrics.ForceMergeActionMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.actionmetrics.MoveShardActionMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.actionmetrics.NotificationActionMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.actionmetrics.OpenActionMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.actionmetrics.ReplicaCountActionMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.actionmetrics.RolloverActionMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.actionmetrics.SetIndexPriorityActionMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.actionmetrics.SetReadOnlyActionMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.actionmetrics.ShrinkActionMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.actionmetrics.SnapshotActionMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.actionmetrics.TransitionActionMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData +import org.opensearch.telemetry.metrics.MetricsRegistry +import org.opensearch.telemetry.metrics.tags.Tags + +abstract class ActionMetrics { + abstract val actionName: String + + fun createTags(context: StepContext): Tags { + val tags = Tags.create() + .addTag("index_name", context.metadata.index) + .addTag("policy_id", context.metadata.policyID) + .addTag("node_id", context.clusterService.nodeName ?: "") + .addTag("index_uuid", context.metadata.indexUuid) + return tags + } + + abstract fun emitMetrics( + context: StepContext, + indexManagementActionsMetrics: IndexManagementActionsMetrics, + stepMetaData: StepMetaData?, + ) +} + +class IndexManagementActionsMetrics private constructor() { + private lateinit var metricsRegistry: MetricsRegistry + private lateinit var actionMetricsMap: Map + + companion object { + val instance: IndexManagementActionsMetrics by lazy { HOLDER.instance } + + const val ROLLOVER = "rollover" + const val NOTIFICATION = "notification" + const val FORCE_MERGE = "force_merge" + const val DELETE = "delete" + const val REPLICA_COUNT = "replica_count" + const val TRANSITION = "transition" + const val CLOSE = "close" + const val SET_INDEX_PRIORITY = "set_index_priority" + const val OPEN = "open" + const val MOVE_SHARD = "move_shard" + const val SET_READ_ONLY = "set_read_only" + const val SHRINK = "shrink" + const val SNAPSHOT = "snapshot" + const val ALIAS_ACTION = "alias_action" + const val ALLOCATION = "allocation" + + private object HOLDER { + val instance = IndexManagementActionsMetrics() + } + } + + fun initialize(metricsRegistry: MetricsRegistry) { + this.metricsRegistry = metricsRegistry + + RolloverActionMetrics.instance.initializeCounters(metricsRegistry) + NotificationActionMetrics.instance.initializeCounters(metricsRegistry) + ForceMergeActionMetrics.instance.initializeCounters(metricsRegistry) + DeleteActionMetrics.instance.initializeCounters(metricsRegistry) + ReplicaCountActionMetrics.instance.initializeCounters(metricsRegistry) + TransitionActionMetrics.instance.initializeCounters(metricsRegistry) + CloseActionMetrics.instance.initializeCounters(metricsRegistry) + SetIndexPriorityActionMetrics.instance.initializeCounters(metricsRegistry) + OpenActionMetrics.instance.initializeCounters(metricsRegistry) + MoveShardActionMetrics.instance.initializeCounters(metricsRegistry) + SetReadOnlyActionMetrics.instance.initializeCounters(metricsRegistry) + ShrinkActionMetrics.instance.initializeCounters(metricsRegistry) + SnapshotActionMetrics.instance.initializeCounters(metricsRegistry) + AliasActionMetrics.instance.initializeCounters(metricsRegistry) + AllocationActionMetrics.instance.initializeCounters(metricsRegistry) + + actionMetricsMap = mapOf( + ROLLOVER to RolloverActionMetrics.instance, + NOTIFICATION to NotificationActionMetrics.instance, + FORCE_MERGE to ForceMergeActionMetrics.instance, + DELETE to DeleteActionMetrics.instance, + REPLICA_COUNT to ReplicaCountActionMetrics.instance, + TRANSITION to TransitionActionMetrics.instance, + CLOSE to CloseActionMetrics.instance, + SET_INDEX_PRIORITY to SetIndexPriorityActionMetrics.instance, + OPEN to OpenActionMetrics.instance, + MOVE_SHARD to MoveShardActionMetrics.instance, + SET_READ_ONLY to SetReadOnlyActionMetrics.instance, + SHRINK to ShrinkActionMetrics.instance, + SNAPSHOT to SnapshotActionMetrics.instance, + ALIAS_ACTION to AliasActionMetrics.instance, + ALLOCATION to AllocationActionMetrics.instance, + ) + } + + fun getActionMetrics(actionName: String): ActionMetrics? { + return actionMetricsMap[actionName] + } +} diff --git a/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/AliasActionMetrics.kt b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/AliasActionMetrics.kt new file mode 100644 index 000000000..5652a292e --- /dev/null +++ b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/AliasActionMetrics.kt @@ -0,0 +1,53 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.actionmetrics + +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step.StepStatus +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.ActionMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.IndexManagementActionsMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData +import org.opensearch.telemetry.metrics.Counter +import org.opensearch.telemetry.metrics.MetricsRegistry + +class AliasActionMetrics private constructor() : ActionMetrics() { + override val actionName: String = IndexManagementActionsMetrics.ALIAS_ACTION + lateinit var successes: Counter + lateinit var failures: Counter + lateinit var cumulativeLatency: Counter + + fun initializeCounters(metricsRegistry: MetricsRegistry) { + successes = metricsRegistry.createCounter("${actionName}_successes", "Alias Action Successes", "count") + failures = metricsRegistry.createCounter("${actionName}_failures", "Alias Action Failures", "count") + cumulativeLatency = metricsRegistry.createCounter("${actionName}_cumulative_latency", "Cumulative Latency of Alias Actions", "milliseconds") + } + + companion object { + val instance: AliasActionMetrics by lazy { HOLDER.instance } + } + + private object HOLDER { + val instance = AliasActionMetrics() + } + + override fun emitMetrics( + context: StepContext, + indexManagementActionsMetrics: IndexManagementActionsMetrics, + stepMetaData: StepMetaData?, + ) { + val aliasActionMetrics = indexManagementActionsMetrics.getActionMetrics(IndexManagementActionsMetrics.ALIAS_ACTION) as AliasActionMetrics + val stepStatus = stepMetaData?.stepStatus + if (stepStatus == StepStatus.COMPLETED) { + aliasActionMetrics.successes.add(1.0, context.let { aliasActionMetrics.createTags(it) }) + } + if (stepStatus == StepStatus.FAILED) { + aliasActionMetrics.failures.add(1.0, context.let { aliasActionMetrics.createTags(it) }) + } + val endTime = System.currentTimeMillis() + val latency = endTime - (context.metadata.stepMetaData?.startTime ?: endTime) + aliasActionMetrics.cumulativeLatency.add(latency.toDouble(), context.let { aliasActionMetrics.createTags(it) }) + } +} diff --git a/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/AllocationActionMetrics.kt b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/AllocationActionMetrics.kt new file mode 100644 index 000000000..363fde91d --- /dev/null +++ b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/AllocationActionMetrics.kt @@ -0,0 +1,53 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.actionmetrics + +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step.StepStatus +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.ActionMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.IndexManagementActionsMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData +import org.opensearch.telemetry.metrics.Counter +import org.opensearch.telemetry.metrics.MetricsRegistry + +class AllocationActionMetrics private constructor() : ActionMetrics() { + override val actionName: String = IndexManagementActionsMetrics.ALLOCATION + lateinit var successes: Counter + lateinit var failures: Counter + lateinit var cumulativeLatency: Counter + + fun initializeCounters(metricsRegistry: MetricsRegistry) { + successes = metricsRegistry.createCounter("${actionName}_successes", "Allocation Action Successes", "count") + failures = metricsRegistry.createCounter("${actionName}_failures", "Allocation Action Failures", "count") + cumulativeLatency = metricsRegistry.createCounter("${actionName}_cumulative_latency", "Cumulative Latency of Allocation Actions", "milliseconds") + } + + companion object { + val instance: AllocationActionMetrics by lazy { HOLDER.instance } + } + + private object HOLDER { + val instance = AllocationActionMetrics() + } + + override fun emitMetrics( + context: StepContext, + indexManagementActionsMetrics: IndexManagementActionsMetrics, + stepMetaData: StepMetaData?, + ) { + val allocationActionMetrics = indexManagementActionsMetrics.getActionMetrics(IndexManagementActionsMetrics.ALLOCATION) as AllocationActionMetrics + val stepStatus = stepMetaData?.stepStatus + if (stepStatus == StepStatus.COMPLETED) { + allocationActionMetrics.successes.add(1.0, context.let { allocationActionMetrics.createTags(it) }) + } + if (stepStatus == StepStatus.FAILED) { + allocationActionMetrics.failures.add(1.0, context.let { allocationActionMetrics.createTags(it) }) + } + val endTime = System.currentTimeMillis() + val latency = endTime - (context.metadata.stepMetaData?.startTime ?: endTime) + allocationActionMetrics.cumulativeLatency.add(latency.toDouble(), context.let { allocationActionMetrics.createTags(it) }) + } +} diff --git a/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/CloseActionMetrics.kt b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/CloseActionMetrics.kt new file mode 100644 index 000000000..1f301872d --- /dev/null +++ b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/CloseActionMetrics.kt @@ -0,0 +1,53 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.actionmetrics + +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step.StepStatus +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.ActionMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.IndexManagementActionsMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData +import org.opensearch.telemetry.metrics.Counter +import org.opensearch.telemetry.metrics.MetricsRegistry + +class CloseActionMetrics private constructor() : ActionMetrics() { + override val actionName: String = IndexManagementActionsMetrics.CLOSE + lateinit var successes: Counter + lateinit var failures: Counter + lateinit var cumulativeLatency: Counter + + fun initializeCounters(metricsRegistry: MetricsRegistry) { + successes = metricsRegistry.createCounter("${actionName}_successes", "Close Action Successes", "count") + failures = metricsRegistry.createCounter("${actionName}_failures", "Close Action Failures", "count") + cumulativeLatency = metricsRegistry.createCounter("${actionName}_cumulative_latency", "Cumulative Latency of Close Actions", "milliseconds") + } + + companion object { + val instance: CloseActionMetrics by lazy { HOLDER.instance } + } + + private object HOLDER { + val instance = CloseActionMetrics() + } + + override fun emitMetrics( + context: StepContext, + indexManagementActionsMetrics: IndexManagementActionsMetrics, + stepMetaData: StepMetaData?, + ) { + val closeActionMetrics = indexManagementActionsMetrics.getActionMetrics(IndexManagementActionsMetrics.CLOSE) as CloseActionMetrics + val stepStatus = stepMetaData?.stepStatus + if (stepStatus == StepStatus.COMPLETED) { + closeActionMetrics.successes.add(1.0, context.let { closeActionMetrics.createTags(it) }) + } + if (stepStatus == StepStatus.FAILED) { + closeActionMetrics.failures.add(1.0, context.let { closeActionMetrics.createTags(it) }) + } + val endTime = System.currentTimeMillis() + val latency = endTime - (context.metadata.stepMetaData?.startTime ?: endTime) + closeActionMetrics.cumulativeLatency.add(latency.toDouble(), context.let { closeActionMetrics.createTags(it) }) + } +} diff --git a/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/DeleteActionMetrics.kt b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/DeleteActionMetrics.kt new file mode 100644 index 000000000..b8d236364 --- /dev/null +++ b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/DeleteActionMetrics.kt @@ -0,0 +1,53 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.actionmetrics + +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step.StepStatus +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.ActionMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.IndexManagementActionsMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData +import org.opensearch.telemetry.metrics.Counter +import org.opensearch.telemetry.metrics.MetricsRegistry + +class DeleteActionMetrics private constructor() : ActionMetrics() { + override val actionName: String = IndexManagementActionsMetrics.DELETE + lateinit var successes: Counter + lateinit var failures: Counter + lateinit var cumulativeLatency: Counter + + fun initializeCounters(metricsRegistry: MetricsRegistry) { + successes = metricsRegistry.createCounter("${actionName}_successes", "Delete Action Successes", "count") + failures = metricsRegistry.createCounter("${actionName}_failures", "Delete Action Failures", "count") + cumulativeLatency = metricsRegistry.createCounter("${actionName}_cumulative_latency", "Cumulative Latency of Delete Action", "milliseconds") + } + + companion object { + val instance: DeleteActionMetrics by lazy { HOLDER.instance } + } + + private object HOLDER { + val instance = DeleteActionMetrics() + } + + override fun emitMetrics( + context: StepContext, + indexManagementActionsMetrics: IndexManagementActionsMetrics, + stepMetaData: StepMetaData?, + ) { + val deleteActionMetrics = indexManagementActionsMetrics.getActionMetrics(IndexManagementActionsMetrics.DELETE) as DeleteActionMetrics + val stepStatus = stepMetaData?.stepStatus + if (stepStatus == StepStatus.COMPLETED) { + deleteActionMetrics.successes.add(1.0, context.let { deleteActionMetrics.createTags(it) }) + } + if (stepStatus == StepStatus.FAILED) { + deleteActionMetrics.failures.add(1.0, context.let { deleteActionMetrics.createTags(it) }) + } + val endTime = System.currentTimeMillis() + val latency = endTime - (context.metadata.stepMetaData?.startTime ?: endTime) + deleteActionMetrics.cumulativeLatency.add(latency.toDouble(), context.let { deleteActionMetrics.createTags(it) }) + } +} diff --git a/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/ForceMergeActionMetrics.kt b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/ForceMergeActionMetrics.kt new file mode 100644 index 000000000..5b7858ee1 --- /dev/null +++ b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/ForceMergeActionMetrics.kt @@ -0,0 +1,53 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.actionmetrics + +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step.StepStatus +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.ActionMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.IndexManagementActionsMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData +import org.opensearch.telemetry.metrics.Counter +import org.opensearch.telemetry.metrics.MetricsRegistry + +class ForceMergeActionMetrics private constructor() : ActionMetrics() { + override val actionName: String = IndexManagementActionsMetrics.FORCE_MERGE + lateinit var successes: Counter + lateinit var failures: Counter + lateinit var cumulativeLatency: Counter + + fun initializeCounters(metricsRegistry: MetricsRegistry) { + successes = metricsRegistry.createCounter("${actionName}_successes", "Force Merge Action Successes", "count") + failures = metricsRegistry.createCounter("${actionName}_failures", "Force Merge Action Failures", "count") + cumulativeLatency = metricsRegistry.createCounter("${actionName}_cumulative_latency", "Cumulative Latency of Force Merge Action", "milliseconds") + } + + companion object { + val instance: ForceMergeActionMetrics by lazy { HOLDER.instance } + } + + private object HOLDER { + val instance = ForceMergeActionMetrics() + } + + override fun emitMetrics( + context: StepContext, + indexManagementActionsMetrics: IndexManagementActionsMetrics, + stepMetaData: StepMetaData?, + ) { + val forceMergeActionMetrics = indexManagementActionsMetrics.getActionMetrics(IndexManagementActionsMetrics.FORCE_MERGE) as ForceMergeActionMetrics + val stepStatus = stepMetaData?.stepStatus + if (stepStatus == StepStatus.COMPLETED) { + forceMergeActionMetrics.successes.add(1.0, context.let { forceMergeActionMetrics.createTags(it) }) + } + if (stepStatus == StepStatus.FAILED) { + forceMergeActionMetrics.failures.add(1.0, context.let { forceMergeActionMetrics.createTags(it) }) + } + val endTime = System.currentTimeMillis() + val latency = endTime - (context.metadata.stepMetaData?.startTime ?: endTime) + forceMergeActionMetrics.cumulativeLatency.add(latency.toDouble(), context.let { forceMergeActionMetrics.createTags(it) }) + } +} diff --git a/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/MoveShardActionMetrics.kt b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/MoveShardActionMetrics.kt new file mode 100644 index 000000000..2f86100d9 --- /dev/null +++ b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/MoveShardActionMetrics.kt @@ -0,0 +1,53 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.actionmetrics + +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step.StepStatus +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.ActionMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.IndexManagementActionsMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData +import org.opensearch.telemetry.metrics.Counter +import org.opensearch.telemetry.metrics.MetricsRegistry + +class MoveShardActionMetrics private constructor() : ActionMetrics() { + override val actionName: String = IndexManagementActionsMetrics.MOVE_SHARD + lateinit var successes: Counter + lateinit var failures: Counter + lateinit var cumulativeLatency: Counter + + fun initializeCounters(metricsRegistry: MetricsRegistry) { + successes = metricsRegistry.createCounter("${actionName}_successes", "Move Shard Action Successes", "count") + failures = metricsRegistry.createCounter("${actionName}_failures", "Move Shard Action Failures", "count") + cumulativeLatency = metricsRegistry.createCounter("${actionName}_cumulative_latency", "Cumulative Latency of Move Shard Actions", "milliseconds") + } + + companion object { + val instance: MoveShardActionMetrics by lazy { HOLDER.instance } + } + + private object HOLDER { + val instance = MoveShardActionMetrics() + } + + override fun emitMetrics( + context: StepContext, + indexManagementActionsMetrics: IndexManagementActionsMetrics, + stepMetaData: StepMetaData?, + ) { + val moveShardActionMetrics = indexManagementActionsMetrics.getActionMetrics(IndexManagementActionsMetrics.MOVE_SHARD) as MoveShardActionMetrics + val stepStatus = stepMetaData?.stepStatus + if (stepStatus == StepStatus.COMPLETED) { + moveShardActionMetrics.successes.add(1.0, context.let { moveShardActionMetrics.createTags(it) }) + } + if (stepStatus == StepStatus.FAILED) { + moveShardActionMetrics.failures.add(1.0, context.let { moveShardActionMetrics.createTags(it) }) + } + val endTime = System.currentTimeMillis() + val latency = endTime - (context.metadata.stepMetaData?.startTime ?: endTime) + moveShardActionMetrics.cumulativeLatency.add(latency.toDouble(), context.let { moveShardActionMetrics.createTags(it) }) + } +} diff --git a/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/NotificationActionMetrics.kt b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/NotificationActionMetrics.kt new file mode 100644 index 000000000..0faa9b9fd --- /dev/null +++ b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/NotificationActionMetrics.kt @@ -0,0 +1,53 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.actionmetrics + +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step.StepStatus +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.ActionMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.IndexManagementActionsMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData +import org.opensearch.telemetry.metrics.Counter +import org.opensearch.telemetry.metrics.MetricsRegistry + +class NotificationActionMetrics private constructor() : ActionMetrics() { + override val actionName: String = IndexManagementActionsMetrics.NOTIFICATION + lateinit var successes: Counter + lateinit var failures: Counter + lateinit var cumulativeLatency: Counter + + fun initializeCounters(metricsRegistry: MetricsRegistry) { + successes = metricsRegistry.createCounter("${actionName}_successes", "Notification Action Successes", "count") + failures = metricsRegistry.createCounter("${actionName}_failures", "Notification Action Failures", "count") + cumulativeLatency = metricsRegistry.createCounter("${actionName}_cumulative_latency", "Cumulative Latency of Notification Action", "milliseconds") + } + + companion object { + val instance: NotificationActionMetrics by lazy { HOLDER.instance } + } + + private object HOLDER { + val instance = NotificationActionMetrics() + } + + override fun emitMetrics( + context: StepContext, + indexManagementActionsMetrics: IndexManagementActionsMetrics, + stepMetaData: StepMetaData?, + ) { + val notificationActionMetrics = indexManagementActionsMetrics.getActionMetrics(IndexManagementActionsMetrics.NOTIFICATION) as NotificationActionMetrics + val stepStatus = stepMetaData?.stepStatus + if (stepStatus == StepStatus.COMPLETED) { + notificationActionMetrics.successes.add(1.0, context.let { notificationActionMetrics.createTags(it) }) + } + if (stepStatus == StepStatus.FAILED) { + notificationActionMetrics.failures.add(1.0, context.let { notificationActionMetrics.createTags(it) }) + } + val endTime = System.currentTimeMillis() + val latency = endTime - (context.metadata.stepMetaData?.startTime ?: endTime) + notificationActionMetrics.cumulativeLatency.add(latency.toDouble(), context.let { notificationActionMetrics.createTags(it) }) + } +} diff --git a/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/OpenActionMetrics.kt b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/OpenActionMetrics.kt new file mode 100644 index 000000000..ad29a5f8a --- /dev/null +++ b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/OpenActionMetrics.kt @@ -0,0 +1,53 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.actionmetrics + +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step.StepStatus +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.ActionMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.IndexManagementActionsMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData +import org.opensearch.telemetry.metrics.Counter +import org.opensearch.telemetry.metrics.MetricsRegistry + +class OpenActionMetrics private constructor() : ActionMetrics() { + override val actionName: String = IndexManagementActionsMetrics.OPEN + lateinit var successes: Counter + lateinit var failures: Counter + lateinit var cumulativeLatency: Counter + + fun initializeCounters(metricsRegistry: MetricsRegistry) { + successes = metricsRegistry.createCounter("${actionName}_successes", "Open Action Successes", "count") + failures = metricsRegistry.createCounter("${actionName}_failures", "Open Action Failures", "count") + cumulativeLatency = metricsRegistry.createCounter("${actionName}_cumulative_latency", "Cumulative Latency of Open Actions", "milliseconds") + } + + companion object { + val instance: OpenActionMetrics by lazy { HOLDER.instance } + } + + private object HOLDER { + val instance = OpenActionMetrics() + } + + override fun emitMetrics( + context: StepContext, + indexManagementActionsMetrics: IndexManagementActionsMetrics, + stepMetaData: StepMetaData?, + ) { + val openActionMetrics = indexManagementActionsMetrics.getActionMetrics(IndexManagementActionsMetrics.OPEN) as OpenActionMetrics + val stepStatus = stepMetaData?.stepStatus + if (stepStatus == StepStatus.COMPLETED) { + openActionMetrics.successes.add(1.0, context.let { openActionMetrics.createTags(it) }) + } + if (stepStatus == StepStatus.FAILED) { + openActionMetrics.failures.add(1.0, context.let { openActionMetrics.createTags(it) }) + } + val endTime = System.currentTimeMillis() + val latency = endTime - (context.metadata.stepMetaData?.startTime ?: endTime) + openActionMetrics.cumulativeLatency.add(latency.toDouble(), context.let { openActionMetrics.createTags(it) }) + } +} diff --git a/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/ReplicaCountActionMetrics.kt b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/ReplicaCountActionMetrics.kt new file mode 100644 index 000000000..94e60b892 --- /dev/null +++ b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/ReplicaCountActionMetrics.kt @@ -0,0 +1,53 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.actionmetrics + +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step.StepStatus +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.ActionMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.IndexManagementActionsMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData +import org.opensearch.telemetry.metrics.Counter +import org.opensearch.telemetry.metrics.MetricsRegistry + +class ReplicaCountActionMetrics private constructor() : ActionMetrics() { + override val actionName: String = IndexManagementActionsMetrics.REPLICA_COUNT + lateinit var successes: Counter + lateinit var failures: Counter + lateinit var cumulativeLatency: Counter + + fun initializeCounters(metricsRegistry: MetricsRegistry) { + successes = metricsRegistry.createCounter("${actionName}_successes", "Replica Action Successes", "count") + failures = metricsRegistry.createCounter("${actionName}_failures", "Replica Action Failures", "count") + cumulativeLatency = metricsRegistry.createCounter("${actionName}_cumulative_latency", "Cumulative Latency of Replica Count Action", "milliseconds") + } + + companion object { + val instance: ReplicaCountActionMetrics by lazy { HOLDER.instance } + } + + private object HOLDER { + val instance = ReplicaCountActionMetrics() + } + + override fun emitMetrics( + context: StepContext, + indexManagementActionsMetrics: IndexManagementActionsMetrics, + stepMetaData: StepMetaData?, + ) { + val replicaCountActionMetrics = indexManagementActionsMetrics.getActionMetrics(IndexManagementActionsMetrics.REPLICA_COUNT) as ReplicaCountActionMetrics + val stepStatus = stepMetaData?.stepStatus + if (stepStatus == StepStatus.COMPLETED) { + replicaCountActionMetrics.successes.add(1.0, context.let { replicaCountActionMetrics.createTags(it) }) + } + if (stepStatus == StepStatus.FAILED) { + replicaCountActionMetrics.failures.add(1.0, context.let { replicaCountActionMetrics.createTags(it) }) + } + val endTime = System.currentTimeMillis() + val latency = endTime - (context.metadata.stepMetaData?.startTime ?: endTime) + replicaCountActionMetrics.cumulativeLatency.add(latency.toDouble(), context.let { replicaCountActionMetrics.createTags(it) }) + } +} diff --git a/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/RolloverActionMetrics.kt b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/RolloverActionMetrics.kt new file mode 100644 index 000000000..3d2965702 --- /dev/null +++ b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/RolloverActionMetrics.kt @@ -0,0 +1,53 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.actionmetrics + +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step.StepStatus +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.ActionMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.IndexManagementActionsMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData +import org.opensearch.telemetry.metrics.Counter +import org.opensearch.telemetry.metrics.MetricsRegistry + +class RolloverActionMetrics private constructor() : ActionMetrics() { + override val actionName: String = IndexManagementActionsMetrics.ROLLOVER + lateinit var successes: Counter + lateinit var failures: Counter + lateinit var cumulativeLatency: Counter + + fun initializeCounters(metricsRegistry: MetricsRegistry) { + successes = metricsRegistry.createCounter("${actionName}_successes", "Rollover Action Successes", "count") + failures = metricsRegistry.createCounter("${actionName}_failures", "Rollover Action Failures", "count") + cumulativeLatency = metricsRegistry.createCounter("${actionName}_cumulative_latency", "Cumulative Latency of Rollover Actions", "milliseconds") + } + + companion object { + val instance: RolloverActionMetrics by lazy { HOLDER.instance } + } + + private object HOLDER { + val instance = RolloverActionMetrics() + } + + override fun emitMetrics( + context: StepContext, + indexManagementActionsMetrics: IndexManagementActionsMetrics, + stepMetaData: StepMetaData?, + ) { + val rolloverActionMetrics = indexManagementActionsMetrics.getActionMetrics(IndexManagementActionsMetrics.ROLLOVER) as RolloverActionMetrics + val stepStatus = stepMetaData?.stepStatus + if (stepStatus == StepStatus.COMPLETED) { + rolloverActionMetrics.successes.add(1.0, context.let { rolloverActionMetrics.createTags(it) }) + } + if (stepStatus == StepStatus.FAILED) { + rolloverActionMetrics.failures.add(1.0, context.let { rolloverActionMetrics.createTags(it) }) + } + val endTime = System.currentTimeMillis() + val latency = endTime - (context.metadata.stepMetaData?.startTime ?: endTime) + rolloverActionMetrics.cumulativeLatency.add(latency.toDouble(), context.let { rolloverActionMetrics.createTags(it) }) + } +} diff --git a/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/SetIndexPriorityActionMetrics.kt b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/SetIndexPriorityActionMetrics.kt new file mode 100644 index 000000000..023461829 --- /dev/null +++ b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/SetIndexPriorityActionMetrics.kt @@ -0,0 +1,53 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.actionmetrics + +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step.StepStatus +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.ActionMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.IndexManagementActionsMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData +import org.opensearch.telemetry.metrics.Counter +import org.opensearch.telemetry.metrics.MetricsRegistry + +class SetIndexPriorityActionMetrics private constructor() : ActionMetrics() { + override val actionName: String = IndexManagementActionsMetrics.SET_INDEX_PRIORITY + lateinit var successes: Counter + lateinit var failures: Counter + lateinit var cumulativeLatency: Counter + + fun initializeCounters(metricsRegistry: MetricsRegistry) { + successes = metricsRegistry.createCounter("${actionName}_successes", "Set Index Priority Action Successes", "count") + failures = metricsRegistry.createCounter("${actionName}_failures", "Set Index Priority Action Failures", "count") + cumulativeLatency = metricsRegistry.createCounter("${actionName}_cumulative_latency", "Cumulative Latency of Set Index Priority Actions", "milliseconds") + } + + companion object { + val instance: SetIndexPriorityActionMetrics by lazy { HOLDER.instance } + } + + private object HOLDER { + val instance = SetIndexPriorityActionMetrics() + } + + override fun emitMetrics( + context: StepContext, + indexManagementActionsMetrics: IndexManagementActionsMetrics, + stepMetaData: StepMetaData?, + ) { + val setIndexPriorityActionMetrics = indexManagementActionsMetrics.getActionMetrics(IndexManagementActionsMetrics.SET_INDEX_PRIORITY) as SetIndexPriorityActionMetrics + val stepStatus = stepMetaData?.stepStatus + if (stepStatus == StepStatus.COMPLETED) { + setIndexPriorityActionMetrics.successes.add(1.0, context.let { setIndexPriorityActionMetrics.createTags(it) }) + } + if (stepStatus == StepStatus.FAILED) { + setIndexPriorityActionMetrics.failures.add(1.0, context.let { setIndexPriorityActionMetrics.createTags(it) }) + } + val endTime = System.currentTimeMillis() + val latency = endTime - (context.metadata.stepMetaData?.startTime ?: endTime) + setIndexPriorityActionMetrics.cumulativeLatency.add(latency.toDouble(), context.let { setIndexPriorityActionMetrics.createTags(it) }) + } +} diff --git a/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/SetReadOnlyActionMetrics.kt b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/SetReadOnlyActionMetrics.kt new file mode 100644 index 000000000..a3a8b4737 --- /dev/null +++ b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/SetReadOnlyActionMetrics.kt @@ -0,0 +1,53 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.actionmetrics + +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step.StepStatus +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.ActionMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.IndexManagementActionsMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData +import org.opensearch.telemetry.metrics.Counter +import org.opensearch.telemetry.metrics.MetricsRegistry + +class SetReadOnlyActionMetrics private constructor() : ActionMetrics() { + override val actionName: String = IndexManagementActionsMetrics.SET_READ_ONLY + lateinit var successes: Counter + lateinit var failures: Counter + lateinit var cumulativeLatency: Counter + + fun initializeCounters(metricsRegistry: MetricsRegistry) { + successes = metricsRegistry.createCounter("${actionName}_successes", "Set Read Only Action Successes", "count") + failures = metricsRegistry.createCounter("${actionName}_failures", "Set Read Only Action Failures", "count") + cumulativeLatency = metricsRegistry.createCounter("${actionName}_cumulative_latency", "Cumulative Latency of Set Read Only Actions", "milliseconds") + } + + companion object { + val instance: SetReadOnlyActionMetrics by lazy { HOLDER.instance } + } + + private object HOLDER { + val instance = SetReadOnlyActionMetrics() + } + + override fun emitMetrics( + context: StepContext, + indexManagementActionsMetrics: IndexManagementActionsMetrics, + stepMetaData: StepMetaData?, + ) { + val setReadOnlyActionMetrics = indexManagementActionsMetrics.getActionMetrics(IndexManagementActionsMetrics.SET_READ_ONLY) as SetReadOnlyActionMetrics + val stepStatus = stepMetaData?.stepStatus + if (stepStatus == StepStatus.COMPLETED) { + setReadOnlyActionMetrics.successes.add(1.0, context.let { setReadOnlyActionMetrics.createTags(it) }) + } + if (stepStatus == StepStatus.FAILED) { + setReadOnlyActionMetrics.failures.add(1.0, context.let { setReadOnlyActionMetrics.createTags(it) }) + } + val endTime = System.currentTimeMillis() + val latency = endTime - (context.metadata.stepMetaData?.startTime ?: endTime) + setReadOnlyActionMetrics.cumulativeLatency.add(latency.toDouble(), context.let { setReadOnlyActionMetrics.createTags(it) }) + } +} diff --git a/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/ShrinkActionMetrics.kt b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/ShrinkActionMetrics.kt new file mode 100644 index 000000000..704feca3c --- /dev/null +++ b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/ShrinkActionMetrics.kt @@ -0,0 +1,53 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.actionmetrics + +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step.StepStatus +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.ActionMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.IndexManagementActionsMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData +import org.opensearch.telemetry.metrics.Counter +import org.opensearch.telemetry.metrics.MetricsRegistry + +class ShrinkActionMetrics private constructor() : ActionMetrics() { + override val actionName: String = IndexManagementActionsMetrics.SHRINK + lateinit var successes: Counter + lateinit var failures: Counter + lateinit var cumulativeLatency: Counter + + fun initializeCounters(metricsRegistry: MetricsRegistry) { + successes = metricsRegistry.createCounter("${actionName}_successes", "Shrink Action Successes", "count") + failures = metricsRegistry.createCounter("${actionName}_failures", "Shrink Action Failures", "count") + cumulativeLatency = metricsRegistry.createCounter("${actionName}_cumulative_latency", "Cumulative Latency of Shrink Actions", "milliseconds") + } + + companion object { + val instance: ShrinkActionMetrics by lazy { HOLDER.instance } + } + + private object HOLDER { + val instance = ShrinkActionMetrics() + } + + override fun emitMetrics( + context: StepContext, + indexManagementActionsMetrics: IndexManagementActionsMetrics, + stepMetaData: StepMetaData?, + ) { + val shrinkActionMetrics = indexManagementActionsMetrics.getActionMetrics(IndexManagementActionsMetrics.SHRINK) as ShrinkActionMetrics + val stepStatus = stepMetaData?.stepStatus + if (stepStatus == StepStatus.COMPLETED) { + shrinkActionMetrics.successes.add(1.0, context.let { shrinkActionMetrics.createTags(it) }) + } + if (stepStatus == StepStatus.FAILED) { + shrinkActionMetrics.failures.add(1.0, context.let { shrinkActionMetrics.createTags(it) }) + } + val endTime = System.currentTimeMillis() + val latency = endTime - (context.metadata.stepMetaData?.startTime ?: endTime) + shrinkActionMetrics.cumulativeLatency.add(latency.toDouble(), context.let { shrinkActionMetrics.createTags(it) }) + } +} diff --git a/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/SnapshotActionMetrics.kt b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/SnapshotActionMetrics.kt new file mode 100644 index 000000000..10d74e195 --- /dev/null +++ b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/SnapshotActionMetrics.kt @@ -0,0 +1,53 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.actionmetrics + +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step.StepStatus +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.ActionMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.IndexManagementActionsMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData +import org.opensearch.telemetry.metrics.Counter +import org.opensearch.telemetry.metrics.MetricsRegistry + +class SnapshotActionMetrics private constructor() : ActionMetrics() { + override val actionName: String = IndexManagementActionsMetrics.SNAPSHOT + lateinit var successes: Counter + lateinit var failures: Counter + lateinit var cumulativeLatency: Counter + + fun initializeCounters(metricsRegistry: MetricsRegistry) { + successes = metricsRegistry.createCounter("${actionName}_successes", "Snapshot Action Successes", "count") + failures = metricsRegistry.createCounter("${actionName}_failures", "Snapshot Action Failures", "count") + cumulativeLatency = metricsRegistry.createCounter("${actionName}_cumulative_latency", "Cumulative Latency of Snapshot Actions", "milliseconds") + } + + companion object { + val instance: SnapshotActionMetrics by lazy { HOLDER.instance } + } + + private object HOLDER { + val instance = SnapshotActionMetrics() + } + + override fun emitMetrics( + context: StepContext, + indexManagementActionsMetrics: IndexManagementActionsMetrics, + stepMetaData: StepMetaData?, + ) { + val snapshotActionMetrics = indexManagementActionsMetrics.getActionMetrics(IndexManagementActionsMetrics.SNAPSHOT) as SnapshotActionMetrics + val stepStatus = stepMetaData?.stepStatus + if (stepStatus == StepStatus.COMPLETED) { + snapshotActionMetrics.successes.add(1.0, context.let { snapshotActionMetrics.createTags(it) }) + } + if (stepStatus == StepStatus.FAILED) { + snapshotActionMetrics.failures.add(1.0, context.let { snapshotActionMetrics.createTags(it) }) + } + val endTime = System.currentTimeMillis() + val latency = endTime - (context.metadata.stepMetaData?.startTime ?: endTime) + snapshotActionMetrics.cumulativeLatency.add(latency.toDouble(), context.let { snapshotActionMetrics.createTags(it) }) + } +} diff --git a/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/TransitionActionMetrics.kt b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/TransitionActionMetrics.kt new file mode 100644 index 000000000..8049ab25e --- /dev/null +++ b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/metrics/actionmetrics/TransitionActionMetrics.kt @@ -0,0 +1,53 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.actionmetrics + +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step.StepStatus +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.ActionMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.IndexManagementActionsMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData +import org.opensearch.telemetry.metrics.Counter +import org.opensearch.telemetry.metrics.MetricsRegistry + +class TransitionActionMetrics private constructor() : ActionMetrics() { + override val actionName: String = IndexManagementActionsMetrics.TRANSITION + lateinit var successes: Counter + lateinit var failures: Counter + lateinit var cumulativeLatency: Counter + + fun initializeCounters(metricsRegistry: MetricsRegistry) { + successes = metricsRegistry.createCounter("${actionName}_successes", "Transition Action Successes", "count") + failures = metricsRegistry.createCounter("${actionName}_failures", "Transition Action Failures", "count") + cumulativeLatency = metricsRegistry.createCounter("${actionName}_cumulative_latency", "Cumulative Latency of Transition Actions", "milliseconds") + } + + companion object { + val instance: TransitionActionMetrics by lazy { HOLDER.instance } + } + + private object HOLDER { + val instance = TransitionActionMetrics() + } + + override fun emitMetrics( + context: StepContext, + indexManagementActionsMetrics: IndexManagementActionsMetrics, + stepMetaData: StepMetaData?, + ) { + val transitionActionMetrics = indexManagementActionsMetrics.getActionMetrics(IndexManagementActionsMetrics.TRANSITION) as TransitionActionMetrics + val stepStatus = stepMetaData?.stepStatus + if (stepStatus == StepStatus.COMPLETED) { + transitionActionMetrics.successes.add(1.0, context.let { transitionActionMetrics.createTags(it) }) + } + if (stepStatus == StepStatus.FAILED) { + transitionActionMetrics.failures.add(1.0, context.let { transitionActionMetrics.createTags(it) }) + } + val endTime = System.currentTimeMillis() + val latency = endTime - (context.metadata.stepMetaData?.startTime ?: endTime) + transitionActionMetrics.cumulativeLatency.add(latency.toDouble(), context.let { transitionActionMetrics.createTags(it) }) + } +} diff --git a/src/main/kotlin/org/opensearch/indexmanagement/IndexManagementPlugin.kt b/src/main/kotlin/org/opensearch/indexmanagement/IndexManagementPlugin.kt index 3a67de075..7660d48e1 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/IndexManagementPlugin.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/IndexManagementPlugin.kt @@ -145,6 +145,7 @@ import org.opensearch.indexmanagement.snapshotmanagement.settings.SnapshotManage import org.opensearch.indexmanagement.spi.IndexManagementExtension import org.opensearch.indexmanagement.spi.indexstatemanagement.IndexMetadataService import org.opensearch.indexmanagement.spi.indexstatemanagement.StatusChecker +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.IndexManagementActionsMetrics import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData import org.opensearch.indexmanagement.transform.TargetIndexMappingService import org.opensearch.indexmanagement.transform.TransformRunner @@ -184,10 +185,13 @@ import org.opensearch.plugins.ExtensiblePlugin import org.opensearch.plugins.NetworkPlugin import org.opensearch.plugins.Plugin import org.opensearch.plugins.SystemIndexPlugin +import org.opensearch.plugins.TelemetryAwarePlugin import org.opensearch.repositories.RepositoriesService import org.opensearch.rest.RestController import org.opensearch.rest.RestHandler import org.opensearch.script.ScriptService +import org.opensearch.telemetry.metrics.MetricsRegistry +import org.opensearch.telemetry.tracing.Tracer import org.opensearch.threadpool.ThreadPool import org.opensearch.transport.RemoteClusterService import org.opensearch.transport.TransportInterceptor @@ -196,7 +200,8 @@ import org.opensearch.watcher.ResourceWatcherService import java.util.function.Supplier @Suppress("TooManyFunctions") -class IndexManagementPlugin : JobSchedulerExtension, NetworkPlugin, ActionPlugin, ExtensiblePlugin, SystemIndexPlugin, Plugin() { +class IndexManagementPlugin : JobSchedulerExtension, NetworkPlugin, ActionPlugin, ExtensiblePlugin, SystemIndexPlugin, + TelemetryAwarePlugin, Plugin() { private val logger = LogManager.getLogger(javaClass) lateinit var indexManagementIndices: IndexManagementIndices lateinit var actionValidation: ActionValidation @@ -210,6 +215,7 @@ class IndexManagementPlugin : JobSchedulerExtension, NetworkPlugin, ActionPlugin private val extensions = mutableSetOf() private val extensionCheckerMap = mutableMapOf() lateinit var indexOperationActionFilter: IndexOperationActionFilter + private lateinit var metricsRegistry: MetricsRegistry companion object { const val PLUGINS_BASE_URI = "/_plugins" @@ -374,8 +380,11 @@ class IndexManagementPlugin : JobSchedulerExtension, NetworkPlugin, ActionPlugin namedWriteableRegistry: NamedWriteableRegistry, indexNameExpressionResolver: IndexNameExpressionResolver, repositoriesServiceSupplier: Supplier, + tracer: Tracer, + metricsRegistry: MetricsRegistry, ): Collection { val settings = environment.settings() + this.metricsRegistry = metricsRegistry this.clusterService = clusterService QueryShardContextFactory.init( client, @@ -385,6 +394,8 @@ class IndexManagementPlugin : JobSchedulerExtension, NetworkPlugin, ActionPlugin namedWriteableRegistry, environment, ) + + IndexManagementActionsMetrics.instance.initialize(metricsRegistry) rollupInterceptor = RollupInterceptor(clusterService, settings, indexNameExpressionResolver) val jvmService = JvmService(environment.settings()) val transformRunner = @@ -453,6 +464,7 @@ class IndexManagementPlugin : JobSchedulerExtension, NetworkPlugin, ActionPlugin .registerThreadPool(threadPool) .registerExtensionChecker(extensionChecker) .registerIndexMetadataProvider(indexMetadataProvider) + .registerIndexManagementActionMetrics(IndexManagementActionsMetrics.instance) val managedIndexCoordinator = ManagedIndexCoordinator( diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/ManagedIndexRunner.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/ManagedIndexRunner.kt index ea7ed3765..003a612ce 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/ManagedIndexRunner.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/ManagedIndexRunner.kt @@ -86,6 +86,7 @@ import org.opensearch.indexmanagement.opensearchapi.withClosableContext import org.opensearch.indexmanagement.spi.indexstatemanagement.Action import org.opensearch.indexmanagement.spi.indexstatemanagement.Step import org.opensearch.indexmanagement.spi.indexstatemanagement.Validate +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.IndexManagementActionsMetrics import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionMetaData import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData import org.opensearch.indexmanagement.spi.indexstatemanagement.model.PolicyRetryInfoMetaData @@ -121,6 +122,7 @@ object ManagedIndexRunner : private lateinit var skipExecFlag: SkipExecution private lateinit var threadPool: ThreadPool private lateinit var extensionStatusChecker: ExtensionStatusChecker + lateinit var indexManagementActionMetrics: IndexManagementActionsMetrics private lateinit var indexMetadataProvider: IndexMetadataProvider private var indexStateManagementEnabled: Boolean = DEFAULT_ISM_ENABLED private var validationServiceEnabled: Boolean = DEFAULT_ACTION_VALIDATION_ENABLED @@ -221,6 +223,11 @@ object ManagedIndexRunner : return this } + fun registerIndexManagementActionMetrics(indexManagementActionsMetrics: IndexManagementActionsMetrics): Any { + this.indexManagementActionMetrics = indexManagementActionsMetrics + return this + } + override fun runJob(job: ScheduledJobParameter, context: JobExecutionContext) { if (job !is ManagedIndexConfig) { throw IllegalArgumentException("Invalid job type, found ${job.javaClass.simpleName} with id: ${context.jobId}") @@ -446,7 +453,8 @@ object ManagedIndexRunner : managedIndexConfig.id, settings, threadPool.threadContext, managedIndexConfig.policy.user, ), ) { - step.preExecute(logger, stepContext.getUpdatedContext(startingManagedIndexMetaData)).execute().postExecute(logger) + step.preExecute(logger, stepContext.getUpdatedContext(startingManagedIndexMetaData)).execute() + .postExecute(logger, indexManagementActionMetrics, step, startingManagedIndexMetaData) } var executedManagedIndexMetaData = startingManagedIndexMetaData.getCompletedManagedIndexMetaData(action, step) diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptCloseStepTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptCloseStepTests.kt index a7e2851e6..09fc4af00 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptCloseStepTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptCloseStepTests.kt @@ -9,8 +9,12 @@ import com.nhaarman.mockitokotlin2.any import com.nhaarman.mockitokotlin2.doAnswer import com.nhaarman.mockitokotlin2.doReturn import com.nhaarman.mockitokotlin2.mock +import com.nhaarman.mockitokotlin2.verify import com.nhaarman.mockitokotlin2.whenever import kotlinx.coroutines.runBlocking +import org.junit.Before +import org.mockito.ArgumentMatchers.anyString +import org.mockito.ArgumentMatchers.eq import org.opensearch.action.admin.indices.close.CloseIndexResponse import org.opensearch.client.AdminClient import org.opensearch.client.Client @@ -18,15 +22,23 @@ import org.opensearch.client.IndicesAdminClient import org.opensearch.cluster.service.ClusterService import org.opensearch.common.settings.Settings import org.opensearch.core.action.ActionListener +import org.opensearch.indexmanagement.indexstatemanagement.ManagedIndexRunner import org.opensearch.indexmanagement.indexstatemanagement.step.close.AttemptCloseStep import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.IndexManagementActionsMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.actionmetrics.CloseActionMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionMetaData import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext import org.opensearch.jobscheduler.spi.utils.LockService import org.opensearch.script.ScriptService import org.opensearch.snapshots.SnapshotInProgressException +import org.opensearch.telemetry.metrics.Counter +import org.opensearch.telemetry.metrics.MetricsRegistry +import org.opensearch.telemetry.metrics.tags.Tags import org.opensearch.test.OpenSearchTestCase import org.opensearch.transport.RemoteTransportException +import java.time.Instant import kotlin.IllegalArgumentException class AttemptCloseStepTests : OpenSearchTestCase() { @@ -34,18 +46,32 @@ class AttemptCloseStepTests : OpenSearchTestCase() { private val scriptService: ScriptService = mock() private val settings: Settings = Settings.EMPTY private val lockService: LockService = LockService(mock(), clusterService) + private lateinit var metricsRegistry: MetricsRegistry + private lateinit var closeActionMetrics: CloseActionMetrics + + @Before + fun setup() { + metricsRegistry = mock() + whenever(metricsRegistry.createCounter(anyString(), anyString(), anyString())).thenAnswer { + mock() + } + IndexManagementActionsMetrics.instance.initialize(metricsRegistry) + ManagedIndexRunner.registerIndexManagementActionMetrics(IndexManagementActionsMetrics.instance) + closeActionMetrics = IndexManagementActionsMetrics.instance.getActionMetrics(IndexManagementActionsMetrics.CLOSE) as CloseActionMetrics + } fun `test close step sets step status to completed when successful`() { val closeIndexResponse = CloseIndexResponse(true, true, listOf()) val client = getClient(getAdminClient(getIndicesAdminClient(closeIndexResponse, null))) runBlocking { - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, ActionMetaData("close", Instant.now().toEpochMilli(), 0, false, 1, null, null), null, null, null) val attemptCloseStep = AttemptCloseStep() val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings, lockService) - attemptCloseStep.preExecute(logger, context).execute() + attemptCloseStep.preExecute(logger, context).execute().postExecute(logger, IndexManagementActionsMetrics.instance, attemptCloseStep, managedIndexMetaData) val updatedManagedIndexMetaData = attemptCloseStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) assertEquals("Step status is not COMPLETED", Step.StepStatus.COMPLETED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) + verify(closeActionMetrics.successes).add(eq(1.0), any()) } } @@ -54,12 +80,13 @@ class AttemptCloseStepTests : OpenSearchTestCase() { val client = getClient(getAdminClient(getIndicesAdminClient(closeIndexResponse, null))) runBlocking { - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, ActionMetaData("close", Instant.now().toEpochMilli(), 0, false, 1, null, null), null, null, null) val attemptCloseStep = AttemptCloseStep() val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings, lockService) - attemptCloseStep.preExecute(logger, context).execute() + attemptCloseStep.preExecute(logger, context).execute().postExecute(logger, IndexManagementActionsMetrics.instance, attemptCloseStep, managedIndexMetaData) val updatedManagedIndexMetaData = attemptCloseStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) + verify(closeActionMetrics.failures).add(eq(1.0), any()) } } @@ -68,12 +95,13 @@ class AttemptCloseStepTests : OpenSearchTestCase() { val client = getClient(getAdminClient(getIndicesAdminClient(null, exception))) runBlocking { - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, ActionMetaData("close", Instant.now().toEpochMilli(), 0, false, 1, null, null), null, null, null) val attemptCloseStep = AttemptCloseStep() val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings, lockService) - attemptCloseStep.preExecute(logger, context).execute() + attemptCloseStep.preExecute(logger, context).execute().postExecute(logger, IndexManagementActionsMetrics.instance, attemptCloseStep, managedIndexMetaData) val updatedManagedIndexMetaData = attemptCloseStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) + verify(closeActionMetrics.failures).add(eq(1.0), any()) } } @@ -85,7 +113,7 @@ class AttemptCloseStepTests : OpenSearchTestCase() { val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) val attemptCloseStep = AttemptCloseStep() val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings, lockService) - attemptCloseStep.preExecute(logger, context).execute() + attemptCloseStep.preExecute(logger, context).execute().postExecute(logger, IndexManagementActionsMetrics.instance, attemptCloseStep, managedIndexMetaData) val updatedManagedIndexMetaData = attemptCloseStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) assertEquals("Step status is not CONDITION_NOT_MET", Step.StepStatus.CONDITION_NOT_MET, updatedManagedIndexMetaData.stepMetaData?.stepStatus) } @@ -110,13 +138,14 @@ class AttemptCloseStepTests : OpenSearchTestCase() { val client = getClient(getAdminClient(getIndicesAdminClient(null, exception))) runBlocking { - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, ActionMetaData("close", Instant.now().toEpochMilli(), 0, false, 1, null, null), null, null, null) val attemptCloseStep = AttemptCloseStep() val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings, lockService) - attemptCloseStep.preExecute(logger, context).execute() + attemptCloseStep.preExecute(logger, context).execute().postExecute(logger, IndexManagementActionsMetrics.instance, attemptCloseStep, managedIndexMetaData) val updatedManagedIndexMetaData = attemptCloseStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) assertEquals("Did not get cause from nested exception", "nested", updatedManagedIndexMetaData.info!!["cause"]) + verify(closeActionMetrics.failures).add(eq(1.0), any()) } } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptDeleteStepTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptDeleteStepTests.kt index 4dcd6e2f1..2f99551ab 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptDeleteStepTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptDeleteStepTests.kt @@ -8,8 +8,12 @@ package org.opensearch.indexmanagement.indexstatemanagement.step import com.nhaarman.mockitokotlin2.any import com.nhaarman.mockitokotlin2.doReturn import com.nhaarman.mockitokotlin2.mock +import com.nhaarman.mockitokotlin2.verify import com.nhaarman.mockitokotlin2.whenever import kotlinx.coroutines.runBlocking +import org.junit.Before +import org.mockito.ArgumentMatchers.anyString +import org.mockito.ArgumentMatchers.eq import org.mockito.Mockito.doAnswer import org.opensearch.action.support.master.AcknowledgedResponse import org.opensearch.client.AdminClient @@ -18,32 +22,57 @@ import org.opensearch.client.IndicesAdminClient import org.opensearch.cluster.service.ClusterService import org.opensearch.common.settings.Settings import org.opensearch.core.action.ActionListener +import org.opensearch.indexmanagement.indexstatemanagement.ManagedIndexRunner import org.opensearch.indexmanagement.indexstatemanagement.step.delete.AttemptDeleteStep import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.IndexManagementActionsMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.actionmetrics.DeleteActionMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionMetaData import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext import org.opensearch.jobscheduler.spi.utils.LockService import org.opensearch.script.ScriptService import org.opensearch.snapshots.SnapshotInProgressException +import org.opensearch.telemetry.metrics.Counter +import org.opensearch.telemetry.metrics.MetricsRegistry +import org.opensearch.telemetry.metrics.tags.Tags import org.opensearch.test.OpenSearchTestCase +import java.time.Instant class AttemptDeleteStepTests : OpenSearchTestCase() { private val clusterService: ClusterService = mock() private val scriptService: ScriptService = mock() private val settings: Settings = Settings.EMPTY private val lockService: LockService = LockService(mock(), clusterService) + private lateinit var metricsRegistry: MetricsRegistry + private lateinit var deleteActionMetrics: DeleteActionMetrics + + @Before + fun setup() { + metricsRegistry = mock() + whenever(metricsRegistry.createCounter(anyString(), anyString(), anyString())).thenAnswer { + mock() + } + IndexManagementActionsMetrics.instance.initialize(metricsRegistry) + ManagedIndexRunner.registerIndexManagementActionMetrics(IndexManagementActionsMetrics.instance) + deleteActionMetrics = IndexManagementActionsMetrics.instance.getActionMetrics(IndexManagementActionsMetrics.DELETE) as DeleteActionMetrics + } fun `test delete step sets step status to completed when successful`() { val acknowledgedResponse = AcknowledgedResponse(true) val client = getClient(getAdminClient(getIndicesAdminClient(acknowledgedResponse, null))) runBlocking { - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val managedIndexMetaData = ManagedIndexMetaData( + "test", "indexUuid", "policy_id", null, null, null, null, null, null, null, + ActionMetaData("delete", Instant.now().toEpochMilli(), 0, false, 1, null, null), null, null, null, + ) val attemptDeleteStep = AttemptDeleteStep() val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings, lockService) - attemptDeleteStep.preExecute(logger, context).execute() + attemptDeleteStep.preExecute(logger, context).execute().postExecute(logger, IndexManagementActionsMetrics.instance, attemptDeleteStep, managedIndexMetaData) val updatedManagedIndexMetaData = attemptDeleteStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) assertEquals("Step status is not COMPLETED", Step.StepStatus.COMPLETED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) + verify(deleteActionMetrics.successes).add(eq(1.0), any()) } } @@ -52,12 +81,13 @@ class AttemptDeleteStepTests : OpenSearchTestCase() { val client = getClient(getAdminClient(getIndicesAdminClient(acknowledgedResponse, null))) runBlocking { - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, ActionMetaData("delete", Instant.now().toEpochMilli(), 0, false, 1, null, null), null, null, null) val attemptDeleteStep = AttemptDeleteStep() val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings, lockService) - attemptDeleteStep.preExecute(logger, context).execute() + attemptDeleteStep.preExecute(logger, context).execute().postExecute(logger, IndexManagementActionsMetrics.instance, attemptDeleteStep, managedIndexMetaData) val updatedManagedIndexMetaData = attemptDeleteStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) + verify(deleteActionMetrics.failures).add(eq(1.0), any()) } } @@ -66,13 +96,14 @@ class AttemptDeleteStepTests : OpenSearchTestCase() { val client = getClient(getAdminClient(getIndicesAdminClient(null, exception))) runBlocking { - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, ActionMetaData("delete", Instant.now().toEpochMilli(), 0, false, 1, null, null), null, null, null) val attemptDeleteStep = AttemptDeleteStep() val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings, lockService) - attemptDeleteStep.preExecute(logger, context).execute() + attemptDeleteStep.preExecute(logger, context).execute().postExecute(logger, IndexManagementActionsMetrics.instance, attemptDeleteStep, managedIndexMetaData) val updatedManagedIndexMetaData = attemptDeleteStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) logger.info(updatedManagedIndexMetaData) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) + verify(deleteActionMetrics.failures).add(eq(1.0), any()) } } @@ -81,10 +112,10 @@ class AttemptDeleteStepTests : OpenSearchTestCase() { val client = getClient(getAdminClient(getIndicesAdminClient(null, exception))) runBlocking { - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, ActionMetaData("delete", Instant.now().toEpochMilli(), 0, false, 1, null, null), null, null, null) val attemptDeleteStep = AttemptDeleteStep() val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings, lockService) - attemptDeleteStep.preExecute(logger, context).execute() + attemptDeleteStep.preExecute(logger, context).execute().postExecute(logger, IndexManagementActionsMetrics.instance, attemptDeleteStep, managedIndexMetaData) val updatedManagedIndexMetaData = attemptDeleteStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) assertEquals("Step status is not CONDITION_NOT_MET", Step.StepStatus.CONDITION_NOT_MET, updatedManagedIndexMetaData.stepMetaData?.stepStatus) } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptOpenStepTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptOpenStepTests.kt index c0797e3d9..b0986c7ad 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptOpenStepTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptOpenStepTests.kt @@ -9,8 +9,12 @@ import com.nhaarman.mockitokotlin2.any import com.nhaarman.mockitokotlin2.doAnswer import com.nhaarman.mockitokotlin2.doReturn import com.nhaarman.mockitokotlin2.mock +import com.nhaarman.mockitokotlin2.verify import com.nhaarman.mockitokotlin2.whenever import kotlinx.coroutines.runBlocking +import org.junit.Before +import org.mockito.ArgumentMatchers.anyString +import org.mockito.ArgumentMatchers.eq import org.opensearch.action.admin.indices.open.OpenIndexResponse import org.opensearch.client.AdminClient import org.opensearch.client.Client @@ -18,32 +22,53 @@ import org.opensearch.client.IndicesAdminClient import org.opensearch.cluster.service.ClusterService import org.opensearch.common.settings.Settings import org.opensearch.core.action.ActionListener +import org.opensearch.indexmanagement.indexstatemanagement.ManagedIndexRunner import org.opensearch.indexmanagement.indexstatemanagement.step.open.AttemptOpenStep import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.IndexManagementActionsMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.actionmetrics.OpenActionMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionMetaData import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext import org.opensearch.jobscheduler.spi.utils.LockService import org.opensearch.script.ScriptService +import org.opensearch.telemetry.metrics.Counter +import org.opensearch.telemetry.metrics.MetricsRegistry +import org.opensearch.telemetry.metrics.tags.Tags import org.opensearch.test.OpenSearchTestCase import org.opensearch.transport.RemoteTransportException +import java.time.Instant class AttemptOpenStepTests : OpenSearchTestCase() { private val clusterService: ClusterService = mock() private val scriptService: ScriptService = mock() private val settings: Settings = Settings.EMPTY private val lockService: LockService = LockService(mock(), clusterService) + private lateinit var metricsRegistry: MetricsRegistry + private lateinit var openActionMetrics: OpenActionMetrics + @Before + fun setup() { + metricsRegistry = mock() + whenever(metricsRegistry.createCounter(anyString(), anyString(), anyString())).thenAnswer { + mock() + } + IndexManagementActionsMetrics.instance.initialize(metricsRegistry) + ManagedIndexRunner.registerIndexManagementActionMetrics(IndexManagementActionsMetrics.instance) + openActionMetrics = IndexManagementActionsMetrics.instance.getActionMetrics(IndexManagementActionsMetrics.OPEN) as OpenActionMetrics + } fun `test open step sets step status to failed when not acknowledged`() { val openIndexResponse = OpenIndexResponse(false, false) val client = getClient(getAdminClient(getIndicesAdminClient(openIndexResponse, null))) runBlocking { - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, ActionMetaData("open", Instant.now().toEpochMilli(), 0, false, 1, null, null), null, null, null) val attemptOpenStep = AttemptOpenStep() val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings, lockService) - attemptOpenStep.preExecute(logger, context).execute() + attemptOpenStep.preExecute(logger, context).execute().postExecute(logger, IndexManagementActionsMetrics.instance, attemptOpenStep, managedIndexMetaData) val updatedManagedIndexMetaData = attemptOpenStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) + verify(openActionMetrics.failures).add(eq(1.0), any()) } } @@ -52,12 +77,13 @@ class AttemptOpenStepTests : OpenSearchTestCase() { val client = getClient(getAdminClient(getIndicesAdminClient(null, exception))) runBlocking { - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, ActionMetaData("open", Instant.now().toEpochMilli(), 0, false, 1, null, null), null, null, null) val attemptOpenStep = AttemptOpenStep() val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings, lockService) - attemptOpenStep.preExecute(logger, context).execute() + attemptOpenStep.preExecute(logger, context).execute().postExecute(logger, IndexManagementActionsMetrics.instance, attemptOpenStep, managedIndexMetaData) val updatedManagedIndexMetaData = attemptOpenStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) + verify(openActionMetrics.failures).add(eq(1.0), any()) } } @@ -66,13 +92,14 @@ class AttemptOpenStepTests : OpenSearchTestCase() { val client = getClient(getAdminClient(getIndicesAdminClient(null, exception))) runBlocking { - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, ActionMetaData("open", Instant.now().toEpochMilli(), 0, false, 1, null, null), null, null, null) val attemptOpenStep = AttemptOpenStep() val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings, lockService) - attemptOpenStep.preExecute(logger, context).execute() + attemptOpenStep.preExecute(logger, context).execute().postExecute(logger, IndexManagementActionsMetrics.instance, attemptOpenStep, managedIndexMetaData) val updatedManagedIndexMetaData = attemptOpenStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) assertEquals("Did not get cause from nested exception", "nested", updatedManagedIndexMetaData.info!!["cause"]) + verify(openActionMetrics.failures).add(eq(1.0), any()) } } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptRolloverStepTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptRolloverStepTests.kt index 69f674f12..b720614d6 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptRolloverStepTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptRolloverStepTests.kt @@ -9,9 +9,12 @@ import com.nhaarman.mockitokotlin2.any import com.nhaarman.mockitokotlin2.doAnswer import com.nhaarman.mockitokotlin2.doReturn import com.nhaarman.mockitokotlin2.mock +import com.nhaarman.mockitokotlin2.verify import com.nhaarman.mockitokotlin2.whenever import kotlinx.coroutines.runBlocking import org.junit.Before +import org.mockito.ArgumentMatchers.anyString +import org.mockito.ArgumentMatchers.eq import org.opensearch.action.admin.indices.rollover.RolloverResponse import org.opensearch.action.support.master.AcknowledgedResponse import org.opensearch.client.AdminClient @@ -24,15 +27,23 @@ import org.opensearch.cluster.service.ClusterService import org.opensearch.common.settings.Settings import org.opensearch.core.action.ActionListener import org.opensearch.index.IndexNotFoundException +import org.opensearch.indexmanagement.indexstatemanagement.ManagedIndexRunner import org.opensearch.indexmanagement.indexstatemanagement.action.RolloverAction import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings import org.opensearch.indexmanagement.indexstatemanagement.step.rollover.AttemptRolloverStep import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.IndexManagementActionsMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.actionmetrics.RolloverActionMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionMetaData import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext import org.opensearch.jobscheduler.spi.utils.LockService import org.opensearch.script.ScriptService +import org.opensearch.telemetry.metrics.Counter +import org.opensearch.telemetry.metrics.MetricsRegistry +import org.opensearch.telemetry.metrics.tags.Tags import org.opensearch.test.OpenSearchTestCase +import java.time.Instant class AttemptRolloverStepTests : OpenSearchTestCase() { private val clusterService: ClusterService = mock() @@ -42,9 +53,20 @@ class AttemptRolloverStepTests : OpenSearchTestCase() { private val oldIndexName = "old_index" private val newIndexName = "new_index" val alias = "alias" + private lateinit var metricsRegistry: MetricsRegistry + private lateinit var rolloverActionMetrics: RolloverActionMetrics @Before fun setup() { + // Setup metrics + metricsRegistry = mock() + whenever(metricsRegistry.createCounter(anyString(), anyString(), anyString())).thenAnswer { + mock() + } + IndexManagementActionsMetrics.instance.initialize(metricsRegistry) + ManagedIndexRunner.registerIndexManagementActionMetrics(IndexManagementActionsMetrics.instance) + rolloverActionMetrics = IndexManagementActionsMetrics.instance.getActionMetrics(IndexManagementActionsMetrics.ROLLOVER) as RolloverActionMetrics + // mock rollover target val clusterState: ClusterState = mock() val metadata: Metadata = mock() @@ -78,15 +100,16 @@ class AttemptRolloverStepTests : OpenSearchTestCase() { oldIndexName, "indexUuid", "policy_id", null, null, null, null, null, null, - null, null, null, + null, ActionMetaData("rollover", Instant.now().toEpochMilli(), 0, false, 1, null, null), null, null, null, rolledOverIndexName = newIndexName, ) val attemptRolloverStep = AttemptRolloverStep(rolloverAction) val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings, lockService) - attemptRolloverStep.preExecute(logger, context).execute() + attemptRolloverStep.preExecute(logger, context).execute().postExecute(logger, IndexManagementActionsMetrics.instance, attemptRolloverStep, managedIndexMetaData) val updatedManagedIndexMetaData = attemptRolloverStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) assertEquals("message info is not matched", AttemptRolloverStep.getCopyAliasNotAckMessage(oldIndexName, newIndexName), updatedManagedIndexMetaData.info?.get("message")) + verify(rolloverActionMetrics.failures).add(eq(1.0), any()) } } @@ -103,15 +126,16 @@ class AttemptRolloverStepTests : OpenSearchTestCase() { oldIndexName, "indexUuid", "policy_id", null, null, null, null, null, null, - null, null, null, + null, ActionMetaData("rollover", Instant.now().toEpochMilli(), 0, false, 1, null, null), null, null, null, rolledOverIndexName = newIndexName, ) val attemptRolloverStep = AttemptRolloverStep(rolloverAction) val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings, lockService) - attemptRolloverStep.preExecute(logger, context).execute() + attemptRolloverStep.preExecute(logger, context).execute().postExecute(logger, IndexManagementActionsMetrics.instance, attemptRolloverStep, managedIndexMetaData) val updatedManagedIndexMetaData = attemptRolloverStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) assertEquals("message info is not matched", AttemptRolloverStep.getFailedCopyAliasMessage(oldIndexName, newIndexName), updatedManagedIndexMetaData.info?.get("message")) + verify(rolloverActionMetrics.failures).add(eq(1.0), any()) } } @@ -128,15 +152,16 @@ class AttemptRolloverStepTests : OpenSearchTestCase() { oldIndexName, "indexUuid", "policy_id", null, null, null, null, null, null, - null, null, null, + null, ActionMetaData("rollover", Instant.now().toEpochMilli(), 0, false, 1, null, null), null, null, null, rolledOverIndexName = newIndexName, ) val attemptRolloverStep = AttemptRolloverStep(rolloverAction) val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings, lockService) - attemptRolloverStep.preExecute(logger, context).execute() + attemptRolloverStep.preExecute(logger, context).execute().postExecute(logger, IndexManagementActionsMetrics.instance, attemptRolloverStep, managedIndexMetaData) val updatedManagedIndexMetaData = attemptRolloverStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) assertEquals("message info is not matched", AttemptRolloverStep.getCopyAliasIndexNotFoundMessage(newIndexName), updatedManagedIndexMetaData.info?.get("message")) + verify(rolloverActionMetrics.failures).add(eq(1.0), any()) } } @@ -153,15 +178,16 @@ class AttemptRolloverStepTests : OpenSearchTestCase() { oldIndexName, "indexUuid", "policy_id", null, null, null, null, null, null, - null, null, null, + null, ActionMetaData("rollover", Instant.now().toEpochMilli(), 0, false, 1, null, null), null, null, null, rolledOverIndexName = null, ) val attemptRolloverStep = AttemptRolloverStep(rolloverAction) val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings, lockService) - attemptRolloverStep.preExecute(logger, context).execute() + attemptRolloverStep.preExecute(logger, context).execute().postExecute(logger, IndexManagementActionsMetrics.instance, attemptRolloverStep, managedIndexMetaData) val updatedManagedIndexMetaData = attemptRolloverStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) assertEquals("Step status is not COMPLETED", Step.StepStatus.COMPLETED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) assertEquals("message info is not matched", AttemptRolloverStep.getCopyAliasRolledOverIndexNotFoundMessage(oldIndexName), updatedManagedIndexMetaData.info?.get("message")) + verify(rolloverActionMetrics.successes).add(eq(1.0), any()) } } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptSetIndexPriorityStepTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptSetIndexPriorityStepTests.kt index 6842b4dc4..a2437a67f 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptSetIndexPriorityStepTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptSetIndexPriorityStepTests.kt @@ -11,6 +11,8 @@ import com.nhaarman.mockitokotlin2.doReturn import com.nhaarman.mockitokotlin2.mock import com.nhaarman.mockitokotlin2.whenever import kotlinx.coroutines.runBlocking +import org.junit.Before +import org.mockito.ArgumentMatchers.anyString import org.opensearch.action.support.master.AcknowledgedResponse import org.opensearch.client.AdminClient import org.opensearch.client.Client @@ -18,21 +20,41 @@ import org.opensearch.client.IndicesAdminClient import org.opensearch.cluster.service.ClusterService import org.opensearch.common.settings.Settings import org.opensearch.core.action.ActionListener +import org.opensearch.indexmanagement.indexstatemanagement.ManagedIndexRunner import org.opensearch.indexmanagement.indexstatemanagement.action.IndexPriorityAction import org.opensearch.indexmanagement.indexstatemanagement.step.indexpriority.AttemptSetIndexPriorityStep import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.IndexManagementActionsMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.actionmetrics.SetIndexPriorityActionMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionMetaData import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext import org.opensearch.jobscheduler.spi.utils.LockService import org.opensearch.script.ScriptService +import org.opensearch.telemetry.metrics.Counter +import org.opensearch.telemetry.metrics.MetricsRegistry import org.opensearch.test.OpenSearchTestCase import org.opensearch.transport.RemoteTransportException +import java.time.Instant class AttemptSetIndexPriorityStepTests : OpenSearchTestCase() { private val clusterService: ClusterService = mock() private val scriptService: ScriptService = mock() private val settings: Settings = Settings.EMPTY private val lockService: LockService = LockService(mock(), clusterService) + private lateinit var metricsRegistry: MetricsRegistry + private lateinit var setIndexPriorityActionMetrics: SetIndexPriorityActionMetrics + + @Before + fun setup() { + metricsRegistry = mock() + whenever(metricsRegistry.createCounter(anyString(), anyString(), anyString())).thenAnswer { + mock() + } + IndexManagementActionsMetrics.instance.initialize(metricsRegistry) + ManagedIndexRunner.registerIndexManagementActionMetrics(IndexManagementActionsMetrics.instance) + setIndexPriorityActionMetrics = IndexManagementActionsMetrics.instance.getActionMetrics(IndexManagementActionsMetrics.SET_INDEX_PRIORITY) as SetIndexPriorityActionMetrics + } fun `test set priority step sets step status to completed when successful`() { val acknowledgedResponse = AcknowledgedResponse(true) @@ -40,10 +62,10 @@ class AttemptSetIndexPriorityStepTests : OpenSearchTestCase() { runBlocking { val indexPriorityAction = IndexPriorityAction(50, 0) - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, ActionMetaData("set_index_priority", Instant.now().toEpochMilli(), 0, false, 1, null, null), null, null, null) val attemptSetPriorityStep = AttemptSetIndexPriorityStep(indexPriorityAction) val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings, lockService) - attemptSetPriorityStep.preExecute(logger, context).execute() + attemptSetPriorityStep.preExecute(logger, context).execute().postExecute(logger, IndexManagementActionsMetrics.instance, attemptSetPriorityStep, managedIndexMetaData) val updatedManagedIndexMetaData = attemptSetPriorityStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) assertEquals("Step status is not COMPLETED", Step.StepStatus.COMPLETED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) } @@ -55,10 +77,10 @@ class AttemptSetIndexPriorityStepTests : OpenSearchTestCase() { runBlocking { val indexPriorityAction = IndexPriorityAction(50, 0) - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, ActionMetaData("set_index_priority", Instant.now().toEpochMilli(), 0, false, 1, null, null), null, null, null) val attemptSetPriorityStep = AttemptSetIndexPriorityStep(indexPriorityAction) val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings, lockService) - attemptSetPriorityStep.preExecute(logger, context).execute() + attemptSetPriorityStep.preExecute(logger, context).execute().postExecute(logger, IndexManagementActionsMetrics.instance, attemptSetPriorityStep, managedIndexMetaData) val updatedManagedIndexMetaData = attemptSetPriorityStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) } @@ -70,10 +92,10 @@ class AttemptSetIndexPriorityStepTests : OpenSearchTestCase() { runBlocking { val indexPriorityAction = IndexPriorityAction(50, 0) - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, ActionMetaData("set_index_priority", Instant.now().toEpochMilli(), 0, false, 1, null, null), null, null, null) val attemptSetPriorityStep = AttemptSetIndexPriorityStep(indexPriorityAction) val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings, lockService) - attemptSetPriorityStep.preExecute(logger, context).execute() + attemptSetPriorityStep.preExecute(logger, context).execute().postExecute(logger, IndexManagementActionsMetrics.instance, attemptSetPriorityStep, managedIndexMetaData) val updatedManagedIndexMetaData = attemptSetPriorityStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) logger.info(updatedManagedIndexMetaData) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) @@ -86,10 +108,10 @@ class AttemptSetIndexPriorityStepTests : OpenSearchTestCase() { runBlocking { val indexPriorityAction = IndexPriorityAction(50, 0) - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, ActionMetaData("set_index_priority", Instant.now().toEpochMilli(), 0, false, 1, null, null), null, null, null) val attemptSetPriorityStep = AttemptSetIndexPriorityStep(indexPriorityAction) val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings, lockService) - attemptSetPriorityStep.preExecute(logger, context).execute() + attemptSetPriorityStep.preExecute(logger, context).execute().postExecute(logger, IndexManagementActionsMetrics.instance, attemptSetPriorityStep, managedIndexMetaData) val updatedManagedIndexMetaData = attemptSetPriorityStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) logger.info(updatedManagedIndexMetaData) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptSetReplicaCountStepTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptSetReplicaCountStepTests.kt index 30f2e6639..825695fe3 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptSetReplicaCountStepTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptSetReplicaCountStepTests.kt @@ -8,8 +8,12 @@ package org.opensearch.indexmanagement.indexstatemanagement.step import com.nhaarman.mockitokotlin2.any import com.nhaarman.mockitokotlin2.doReturn import com.nhaarman.mockitokotlin2.mock +import com.nhaarman.mockitokotlin2.verify import com.nhaarman.mockitokotlin2.whenever import kotlinx.coroutines.runBlocking +import org.junit.Before +import org.mockito.ArgumentMatchers.anyString +import org.mockito.ArgumentMatchers.eq import org.mockito.Mockito.doAnswer import org.opensearch.action.support.master.AcknowledgedResponse import org.opensearch.client.AdminClient @@ -18,34 +22,56 @@ import org.opensearch.client.IndicesAdminClient import org.opensearch.cluster.service.ClusterService import org.opensearch.common.settings.Settings import org.opensearch.core.action.ActionListener +import org.opensearch.indexmanagement.indexstatemanagement.ManagedIndexRunner import org.opensearch.indexmanagement.indexstatemanagement.action.ReplicaCountAction import org.opensearch.indexmanagement.indexstatemanagement.step.replicacount.AttemptReplicaCountStep import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.IndexManagementActionsMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.actionmetrics.ReplicaCountActionMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionMetaData import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext import org.opensearch.jobscheduler.spi.utils.LockService import org.opensearch.script.ScriptService +import org.opensearch.telemetry.metrics.Counter +import org.opensearch.telemetry.metrics.MetricsRegistry +import org.opensearch.telemetry.metrics.tags.Tags import org.opensearch.test.OpenSearchTestCase import org.opensearch.transport.RemoteTransportException +import java.time.Instant class AttemptSetReplicaCountStepTests : OpenSearchTestCase() { private val clusterService: ClusterService = mock() private val scriptService: ScriptService = mock() private val settings: Settings = Settings.EMPTY private val lockService: LockService = LockService(mock(), clusterService) + private lateinit var metricsRegistry: MetricsRegistry + private lateinit var replicaCountActionMetrics: ReplicaCountActionMetrics + @Before + fun setup() { + metricsRegistry = mock() + whenever(metricsRegistry.createCounter(anyString(), anyString(), anyString())).thenAnswer { + mock() + } + IndexManagementActionsMetrics.instance.initialize(metricsRegistry) + ManagedIndexRunner.registerIndexManagementActionMetrics(IndexManagementActionsMetrics.instance) + IndexManagementActionsMetrics.instance.initialize(metricsRegistry) + replicaCountActionMetrics = IndexManagementActionsMetrics.instance.getActionMetrics(IndexManagementActionsMetrics.REPLICA_COUNT) as ReplicaCountActionMetrics + } fun `test replica step sets step status to failed when not acknowledged`() { val replicaCountResponse = AcknowledgedResponse(false) val client = getClient(getAdminClient(getIndicesAdminClient(replicaCountResponse, null))) runBlocking { val replicaCountAction = ReplicaCountAction(2, 0) - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) - val replicaCountStep = AttemptReplicaCountStep(replicaCountAction) + val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, ActionMetaData("replica_count", Instant.now().toEpochMilli(), 0, false, 1, null, null), null, null, null) + val attemptReplicaCountStep = AttemptReplicaCountStep(replicaCountAction) val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings, lockService) - replicaCountStep.preExecute(logger, context).execute() - val updatedManagedIndexMetaData = replicaCountStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) + attemptReplicaCountStep.preExecute(logger, context).execute().postExecute(logger, IndexManagementActionsMetrics.instance, attemptReplicaCountStep, managedIndexMetaData) + val updatedManagedIndexMetaData = attemptReplicaCountStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) + verify(replicaCountActionMetrics.failures).add(eq(1.0), any()) } } @@ -55,12 +81,13 @@ class AttemptSetReplicaCountStepTests : OpenSearchTestCase() { runBlocking { val replicaCountAction = ReplicaCountAction(2, 0) - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) - val replicaCountStep = AttemptReplicaCountStep(replicaCountAction) + val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, ActionMetaData("replica_count", Instant.now().toEpochMilli(), 0, false, 1, null, null), null, null, null) + val attemptReplicaCountStep = AttemptReplicaCountStep(replicaCountAction) val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings, lockService) - replicaCountStep.preExecute(logger, context).execute() - val updatedManagedIndexMetaData = replicaCountStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) + attemptReplicaCountStep.preExecute(logger, context).execute().postExecute(logger, IndexManagementActionsMetrics.instance, attemptReplicaCountStep, managedIndexMetaData) + val updatedManagedIndexMetaData = attemptReplicaCountStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) + verify(replicaCountActionMetrics.failures).add(eq(1.0), any()) } } @@ -70,13 +97,14 @@ class AttemptSetReplicaCountStepTests : OpenSearchTestCase() { runBlocking { val replicaCountAction = ReplicaCountAction(2, 0) - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) - val replicaCountStep = AttemptReplicaCountStep(replicaCountAction) + val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, ActionMetaData("replica_count", Instant.now().toEpochMilli(), 0, false, 1, null, null), null, null, null) + val attemptReplicaCountStep = AttemptReplicaCountStep(replicaCountAction) val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings, lockService) - replicaCountStep.preExecute(logger, context).execute() - val updatedManagedIndexMetaData = replicaCountStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) + attemptReplicaCountStep.preExecute(logger, context).execute().postExecute(logger, IndexManagementActionsMetrics.instance, attemptReplicaCountStep, managedIndexMetaData) + val updatedManagedIndexMetaData = attemptReplicaCountStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) assertEquals("Did not get cause from nested exception", "nested", updatedManagedIndexMetaData.info!!["cause"]) + verify(replicaCountActionMetrics.failures).add(eq(1.0), any()) } } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptSnapshotStepTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptSnapshotStepTests.kt index 2cc7040db..34f9c4fa6 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptSnapshotStepTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptSnapshotStepTests.kt @@ -10,9 +10,13 @@ import com.nhaarman.mockitokotlin2.doAnswer import com.nhaarman.mockitokotlin2.doReturn import com.nhaarman.mockitokotlin2.eq import com.nhaarman.mockitokotlin2.mock +import com.nhaarman.mockitokotlin2.verify import com.nhaarman.mockitokotlin2.whenever import kotlinx.coroutines.runBlocking import org.junit.Before +import org.mockito.ArgumentMatchers +import org.mockito.ArgumentMatchers.anyString +import org.mockito.Mockito import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse import org.opensearch.client.AdminClient import org.opensearch.client.Client @@ -22,10 +26,13 @@ import org.opensearch.common.settings.ClusterSettings import org.opensearch.common.settings.Settings import org.opensearch.core.action.ActionListener import org.opensearch.core.rest.RestStatus +import org.opensearch.indexmanagement.indexstatemanagement.ManagedIndexRunner import org.opensearch.indexmanagement.indexstatemanagement.randomSnapshotActionConfig import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings.Companion.SNAPSHOT_DENY_LIST import org.opensearch.indexmanagement.indexstatemanagement.step.snapshot.AttemptSnapshotStep import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.IndexManagementActionsMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.actionmetrics.SnapshotActionMetrics import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionMetaData import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionProperties import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData @@ -35,6 +42,9 @@ import org.opensearch.jobscheduler.spi.utils.LockService import org.opensearch.script.ScriptService import org.opensearch.script.TemplateScript import org.opensearch.snapshots.ConcurrentSnapshotExecutionException +import org.opensearch.telemetry.metrics.Counter +import org.opensearch.telemetry.metrics.MetricsRegistry +import org.opensearch.telemetry.metrics.tags.Tags import org.opensearch.test.OpenSearchTestCase import org.opensearch.transport.RemoteTransportException @@ -43,13 +53,22 @@ class AttemptSnapshotStepTests : OpenSearchTestCase() { private val scriptService: ScriptService = mock() private val settings: Settings = Settings.EMPTY private val snapshotAction = randomSnapshotActionConfig("repo", "snapshot-name") - private val metadata = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, ActionMetaData(AttemptSnapshotStep.name, 1, 0, false, 0, null, ActionProperties(snapshotName = "snapshot-name")), null, null, null) + private val metadata = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, ActionMetaData("snapshot", 1, 0, false, 0, null, ActionProperties(snapshotName = "snapshot-name")), null, null, null) private val lockService: LockService = LockService(mock(), clusterService) + private lateinit var metricsRegistry: MetricsRegistry + private lateinit var snapshotActionMetrics: SnapshotActionMetrics @Before fun settings() { whenever(clusterService.clusterSettings).doReturn(ClusterSettings(Settings.EMPTY, setOf(SNAPSHOT_DENY_LIST))) whenever(scriptService.compile(any(), eq(TemplateScript.CONTEXT))).doReturn(MockTemplateScript.Factory("snapshot-name")) + metricsRegistry = mock() + whenever(metricsRegistry.createCounter(anyString(), anyString(), anyString())).thenAnswer { + mock() + } + IndexManagementActionsMetrics.instance.initialize(metricsRegistry) + ManagedIndexRunner.registerIndexManagementActionMetrics(IndexManagementActionsMetrics.instance) + snapshotActionMetrics = IndexManagementActionsMetrics.instance.getActionMetrics(IndexManagementActionsMetrics.SNAPSHOT) as SnapshotActionMetrics } fun `test snapshot response when block`() { @@ -60,27 +79,30 @@ class AttemptSnapshotStepTests : OpenSearchTestCase() { runBlocking { val step = AttemptSnapshotStep(snapshotAction) val context = StepContext(metadata, clusterService, client, null, null, scriptService, settings, lockService) - step.preExecute(logger, context).execute() + step.preExecute(logger, context).execute().postExecute(logger, IndexManagementActionsMetrics.instance, step, metadata) val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetadata(metadata) assertEquals("Step status is not COMPLETED", Step.StepStatus.COMPLETED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) + verify(snapshotActionMetrics.successes).add(ArgumentMatchers.eq(1.0), any()) } whenever(response.status()).doReturn(RestStatus.OK) runBlocking { val step = AttemptSnapshotStep(snapshotAction) val context = StepContext(metadata, clusterService, client, null, null, scriptService, settings, lockService) - step.preExecute(logger, context).execute() + step.preExecute(logger, context).execute().postExecute(logger, IndexManagementActionsMetrics.instance, step, metadata) val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetadata(metadata) assertEquals("Step status is not COMPLETED", Step.StepStatus.COMPLETED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) + verify(snapshotActionMetrics.successes, Mockito.times(2)).add(ArgumentMatchers.eq(1.0), any()) } whenever(response.status()).doReturn(RestStatus.INTERNAL_SERVER_ERROR) runBlocking { val step = AttemptSnapshotStep(snapshotAction) val context = StepContext(metadata, clusterService, client, null, null, scriptService, settings, lockService) - step.preExecute(logger, context).execute() + step.preExecute(logger, context).execute().postExecute(logger, IndexManagementActionsMetrics.instance, step, metadata) val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetadata(metadata) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) + verify(snapshotActionMetrics.failures).add(ArgumentMatchers.eq(1.0), any()) } } @@ -90,10 +112,11 @@ class AttemptSnapshotStepTests : OpenSearchTestCase() { runBlocking { val step = AttemptSnapshotStep(snapshotAction) val context = StepContext(metadata, clusterService, client, null, null, scriptService, settings, lockService) - step.preExecute(logger, context).execute() + step.preExecute(logger, context).execute().postExecute(logger, IndexManagementActionsMetrics.instance, step, metadata) val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetadata(metadata) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) assertEquals("Did not get cause from nested exception", "example", updatedManagedIndexMetaData.info!!["cause"]) + verify(snapshotActionMetrics.failures).add(ArgumentMatchers.eq(1.0), any()) } } @@ -129,10 +152,11 @@ class AttemptSnapshotStepTests : OpenSearchTestCase() { runBlocking { val step = AttemptSnapshotStep(snapshotAction) val context = StepContext(metadata, clusterService, client, null, null, scriptService, settings, lockService) - step.preExecute(logger, context).execute() + step.preExecute(logger, context).execute().postExecute(logger, IndexManagementActionsMetrics.instance, step, metadata) val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetadata(metadata) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) assertEquals("Did not get cause from nested exception", "some error", updatedManagedIndexMetaData.info!!["cause"]) + verify(snapshotActionMetrics.failures).add(ArgumentMatchers.eq(1.0), any()) } } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptTransitionStepTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptTransitionStepTests.kt index 496c4f3b4..6f6d2cc16 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptTransitionStepTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/AttemptTransitionStepTests.kt @@ -9,9 +9,12 @@ import com.nhaarman.mockitokotlin2.any import com.nhaarman.mockitokotlin2.doAnswer import com.nhaarman.mockitokotlin2.doReturn import com.nhaarman.mockitokotlin2.mock +import com.nhaarman.mockitokotlin2.verify import com.nhaarman.mockitokotlin2.whenever import kotlinx.coroutines.runBlocking import org.junit.Before +import org.mockito.ArgumentMatchers.anyString +import org.mockito.ArgumentMatchers.eq import org.opensearch.action.admin.indices.rollover.RolloverInfo import org.opensearch.action.admin.indices.stats.CommonStats import org.opensearch.action.admin.indices.stats.IndicesStatsResponse @@ -28,17 +31,24 @@ import org.opensearch.core.action.ActionListener import org.opensearch.core.rest.RestStatus import org.opensearch.index.shard.DocsStats import org.opensearch.indexmanagement.indexstatemanagement.IndexMetadataProvider +import org.opensearch.indexmanagement.indexstatemanagement.ManagedIndexRunner import org.opensearch.indexmanagement.indexstatemanagement.action.TransitionsAction import org.opensearch.indexmanagement.indexstatemanagement.model.Conditions import org.opensearch.indexmanagement.indexstatemanagement.model.Transition import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings import org.opensearch.indexmanagement.indexstatemanagement.step.transition.AttemptTransitionStep import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.IndexManagementActionsMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.actionmetrics.TransitionActionMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionMetaData import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData import org.opensearch.jobscheduler.spi.utils.LockService import org.opensearch.script.ScriptService +import org.opensearch.telemetry.metrics.Counter +import org.opensearch.telemetry.metrics.MetricsRegistry +import org.opensearch.telemetry.metrics.tags.Tags import org.opensearch.test.OpenSearchTestCase import org.opensearch.transport.RemoteTransportException import java.time.Instant @@ -46,6 +56,8 @@ import java.time.Instant class AttemptTransitionStepTests : OpenSearchTestCase() { private val indexName: String = "test" private val indexUUID: String = "indexUuid" + private lateinit var metricsRegistry: MetricsRegistry + private lateinit var transitionActionMetrics: TransitionActionMetrics @Suppress("UNCHECKED_CAST") private val indexMetadata: IndexMetadata = @@ -70,6 +82,13 @@ class AttemptTransitionStepTests : OpenSearchTestCase() { @Before fun `setup settings`() { + metricsRegistry = mock() + whenever(metricsRegistry.createCounter(anyString(), anyString(), anyString())).thenAnswer { + mock() + } + IndexManagementActionsMetrics.instance.initialize(metricsRegistry) + ManagedIndexRunner.registerIndexManagementActionMetrics(IndexManagementActionsMetrics.instance) + transitionActionMetrics = IndexManagementActionsMetrics.instance.getActionMetrics(IndexManagementActionsMetrics.TRANSITION) as TransitionActionMetrics whenever(clusterService.clusterSettings).doReturn(ClusterSettings(Settings.EMPTY, setOf(ManagedIndexSettings.RESTRICTED_INDEX_PATTERN))) } @@ -83,14 +102,15 @@ class AttemptTransitionStepTests : OpenSearchTestCase() { val indexMetadataProvider = IndexMetadataProvider(settings, client, clusterService, mutableMapOf()) runBlocking { - val managedIndexMetadata = ManagedIndexMetaData(indexName, indexUUID, "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val managedIndexMetadata = ManagedIndexMetaData(indexName, indexUUID, "policy_id", null, null, null, null, null, null, null, ActionMetaData("transition", Instant.now().toEpochMilli(), 0, false, 1, null, null), null, null, null) val transitionsAction = TransitionsAction(listOf(Transition("some_state", Conditions(docCount = 5L))), indexMetadataProvider) val attemptTransitionStep = AttemptTransitionStep(transitionsAction) val context = StepContext(managedIndexMetadata, clusterService, client, null, null, scriptService, settings, lockService) - attemptTransitionStep.preExecute(logger, context).execute() + attemptTransitionStep.preExecute(logger, context).execute().postExecute(logger, IndexManagementActionsMetrics.instance, attemptTransitionStep, managedIndexMetadata) val updatedManagedIndexMetaData = attemptTransitionStep.getUpdatedManagedIndexMetadata(managedIndexMetadata) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) assertEquals("Did not get correct failed message", AttemptTransitionStep.getFailedStatsMessage(indexName), updatedManagedIndexMetaData.info!!["message"]) + verify(transitionActionMetrics.failures).add(eq(1.0), any()) } } @@ -101,14 +121,15 @@ class AttemptTransitionStepTests : OpenSearchTestCase() { val indexMetadataProvider = IndexMetadataProvider(settings, client, clusterService, mutableMapOf()) runBlocking { - val managedIndexMetadata = ManagedIndexMetaData(indexName, indexUUID, "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val managedIndexMetadata = ManagedIndexMetaData(indexName, indexUUID, "policy_id", null, null, null, null, null, null, null, ActionMetaData("transition", Instant.now().toEpochMilli(), 0, false, 1, null, null), null, null, null) val transitionsAction = TransitionsAction(listOf(Transition("some_state", Conditions(docCount = 5L))), indexMetadataProvider) val attemptTransitionStep = AttemptTransitionStep(transitionsAction) val context = StepContext(managedIndexMetadata, clusterService, client, null, null, scriptService, settings, lockService) - attemptTransitionStep.preExecute(logger, context).execute() + attemptTransitionStep.preExecute(logger, context).execute().postExecute(logger, IndexManagementActionsMetrics.instance, attemptTransitionStep, managedIndexMetadata) val updatedManagedIndexMetaData = attemptTransitionStep.getUpdatedManagedIndexMetadata(managedIndexMetadata) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) assertEquals("Did not get cause from nested exception", "example", updatedManagedIndexMetaData.info!!["cause"]) + verify(transitionActionMetrics.failures).add(eq(1.0), any()) } } @@ -119,14 +140,15 @@ class AttemptTransitionStepTests : OpenSearchTestCase() { val indexMetadataProvider = IndexMetadataProvider(settings, client, clusterService, mutableMapOf()) runBlocking { - val managedIndexMetadata = ManagedIndexMetaData(indexName, indexUUID, "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val managedIndexMetadata = ManagedIndexMetaData(indexName, indexUUID, "policy_id", null, null, null, null, null, null, null, ActionMetaData("transition", Instant.now().toEpochMilli(), 0, false, 1, null, null), null, null, null) val transitionsAction = TransitionsAction(listOf(Transition("some_state", Conditions(docCount = 5L))), indexMetadataProvider) val attemptTransitionStep = AttemptTransitionStep(transitionsAction) val context = StepContext(managedIndexMetadata, clusterService, client, null, null, scriptService, settings, lockService) - attemptTransitionStep.preExecute(logger, context).execute() + attemptTransitionStep.preExecute(logger, context).execute().postExecute(logger, IndexManagementActionsMetrics.instance, attemptTransitionStep, managedIndexMetadata) val updatedManagedIndexMetaData = attemptTransitionStep.getUpdatedManagedIndexMetadata(managedIndexMetadata) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) assertEquals("Did not get cause from nested exception", "nested", updatedManagedIndexMetaData.info!!["cause"]) + verify(transitionActionMetrics.failures).add(eq(1.0), any()) } } @@ -134,7 +156,7 @@ class AttemptTransitionStepTests : OpenSearchTestCase() { val indexMetadataProvider = IndexMetadataProvider(settings, mock(), clusterService, mutableMapOf()) runBlocking { val completedStartTime = Instant.now() - val managedIndexMetadata = ManagedIndexMetaData(indexName, indexUUID, "policy_id", null, null, null, null, null, null, null, null, StepMetaData("attempt_transition", completedStartTime.toEpochMilli(), Step.StepStatus.COMPLETED), null, null) + val managedIndexMetadata = ManagedIndexMetaData(indexName, indexUUID, "policy_id", null, null, null, null, null, null, null, ActionMetaData("transition", Instant.now().toEpochMilli(), 0, false, 1, null, null), StepMetaData("attempt_transition", completedStartTime.toEpochMilli(), Step.StepStatus.COMPLETED), null, null) val transitionsAction = TransitionsAction(listOf(Transition("some_state", null)), indexMetadataProvider) val attemptTransitionStep = AttemptTransitionStep(transitionsAction) Thread.sleep(50) // Make sure we give enough time for the instants to be different diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/SetReadOnlyStepTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/SetReadOnlyStepTests.kt index a3a70ca6c..beed36762 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/SetReadOnlyStepTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/SetReadOnlyStepTests.kt @@ -9,8 +9,12 @@ import com.nhaarman.mockitokotlin2.any import com.nhaarman.mockitokotlin2.doAnswer import com.nhaarman.mockitokotlin2.doReturn import com.nhaarman.mockitokotlin2.mock +import com.nhaarman.mockitokotlin2.verify import com.nhaarman.mockitokotlin2.whenever import kotlinx.coroutines.runBlocking +import org.junit.Before +import org.mockito.ArgumentMatchers.anyString +import org.mockito.ArgumentMatchers.eq import org.opensearch.action.support.master.AcknowledgedResponse import org.opensearch.client.AdminClient import org.opensearch.client.Client @@ -18,32 +22,54 @@ import org.opensearch.client.IndicesAdminClient import org.opensearch.cluster.service.ClusterService import org.opensearch.common.settings.Settings import org.opensearch.core.action.ActionListener +import org.opensearch.indexmanagement.indexstatemanagement.ManagedIndexRunner import org.opensearch.indexmanagement.indexstatemanagement.step.readonly.SetReadOnlyStep import org.opensearch.indexmanagement.spi.indexstatemanagement.Step +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.IndexManagementActionsMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.metrics.actionmetrics.SetReadOnlyActionMetrics +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionMetaData import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext import org.opensearch.jobscheduler.spi.utils.LockService import org.opensearch.script.ScriptService +import org.opensearch.telemetry.metrics.Counter +import org.opensearch.telemetry.metrics.MetricsRegistry +import org.opensearch.telemetry.metrics.tags.Tags import org.opensearch.test.OpenSearchTestCase import org.opensearch.transport.RemoteTransportException +import java.time.Instant class SetReadOnlyStepTests : OpenSearchTestCase() { private val clusterService: ClusterService = mock() private val scriptService: ScriptService = mock() private val settings: Settings = Settings.EMPTY private val lockService: LockService = LockService(mock(), clusterService) + private lateinit var metricsRegistry: MetricsRegistry + private lateinit var setReadOnlyActionMetrics: SetReadOnlyActionMetrics + + @Before + fun setup() { + metricsRegistry = mock() + whenever(metricsRegistry.createCounter(anyString(), anyString(), anyString())).thenAnswer { + mock() + } + IndexManagementActionsMetrics.instance.initialize(metricsRegistry) + ManagedIndexRunner.registerIndexManagementActionMetrics(IndexManagementActionsMetrics.instance) + setReadOnlyActionMetrics = IndexManagementActionsMetrics.instance.getActionMetrics(IndexManagementActionsMetrics.SET_READ_ONLY) as SetReadOnlyActionMetrics + } fun `test read only step sets step status to failed when not acknowledged`() { val setReadOnlyResponse = AcknowledgedResponse(false) val client = getClient(getAdminClient(getIndicesAdminClient(setReadOnlyResponse, null))) runBlocking { - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, ActionMetaData("set_read_only", Instant.now().toEpochMilli(), 0, false, 1, null, null), null, null, null) val setReadOnlyStep = SetReadOnlyStep() val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings, lockService) - setReadOnlyStep.preExecute(logger, context).execute() + setReadOnlyStep.preExecute(logger, context).execute().postExecute(logger, IndexManagementActionsMetrics.instance, setReadOnlyStep, managedIndexMetaData) val updatedManagedIndexMetaData = setReadOnlyStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) + verify(setReadOnlyActionMetrics.failures).add(eq(1.0), any()) } } @@ -52,12 +78,13 @@ class SetReadOnlyStepTests : OpenSearchTestCase() { val client = getClient(getAdminClient(getIndicesAdminClient(null, exception))) runBlocking { - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, ActionMetaData("set_read_only", Instant.now().toEpochMilli(), 0, false, 1, null, null), null, null, null) val setReadOnlyStep = SetReadOnlyStep() val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings, lockService) - setReadOnlyStep.preExecute(logger, context).execute() + setReadOnlyStep.preExecute(logger, context).execute().postExecute(logger, IndexManagementActionsMetrics.instance, setReadOnlyStep, managedIndexMetaData) val updatedManagedIndexMetaData = setReadOnlyStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) + verify(setReadOnlyActionMetrics.failures).add(eq(1.0), any()) } } @@ -66,13 +93,14 @@ class SetReadOnlyStepTests : OpenSearchTestCase() { val client = getClient(getAdminClient(getIndicesAdminClient(null, exception))) runBlocking { - val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val managedIndexMetaData = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, ActionMetaData("set_read_only", Instant.now().toEpochMilli(), 0, false, 1, null, null), null, null, null) val setReadOnlyStep = SetReadOnlyStep() val context = StepContext(managedIndexMetaData, clusterService, client, null, null, scriptService, settings, lockService) - setReadOnlyStep.preExecute(logger, context).execute() + setReadOnlyStep.preExecute(logger, context).execute().postExecute(logger, IndexManagementActionsMetrics.instance, setReadOnlyStep, managedIndexMetaData) val updatedManagedIndexMetaData = setReadOnlyStep.getUpdatedManagedIndexMetadata(managedIndexMetaData) assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) assertEquals("Did not get cause from nested exception", "nested", updatedManagedIndexMetaData.info!!["cause"]) + verify(setReadOnlyActionMetrics.failures).add(eq(1.0), any()) } } From dbd2bc25f5e338819237af5a6b35c6c1e35d7171 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Tue, 9 Jul 2024 11:06:14 -0500 Subject: [PATCH 09/37] Update PULL_REQUEST_TEMPLATE to include an API spec change in the checklist. (#1202) --- .github/PULL_REQUEST_TEMPLATE.md | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index fbc76ee75..61546d170 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,9 +1,16 @@ -*Issue #, if available:* +### Description +[Describe what this change achieves] -*Description of changes:* +### Related Issues +Resolves #[Issue number to be closed when this PR is merged] + -*CheckList:* -- [ ] Commits are signed per the DCO using --signoff +### Check List +- [ ] New functionality includes testing. +- [ ] New functionality has been documented. +- [ ] API changes companion pull request [created](https://github.com/opensearch-project/opensearch-api-specification/blob/main/DEVELOPER_GUIDE.md). +- [ ] Commits are signed per the DCO using `--signoff`. +- [ ] Public documentation issue/PR [created](https://github.com/opensearch-project/documentation-website/issues/new/choose). By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/index-management/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). From 27bc6649c8d3ecdeaf1814c7439adfb33269eaa2 Mon Sep 17 00:00:00 2001 From: bowenlan-amzn Date: Thu, 18 Jul 2024 08:54:38 -0700 Subject: [PATCH 10/37] Fix github action (#1208) --- .github/workflows/bwc-test-workflow.yml | 2 +- .github/workflows/docker-security-test-workflow.yml | 2 +- .github/workflows/multi-node-test-workflow.yml | 2 ++ .github/workflows/security-test-workflow.yml | 4 +++- .github/workflows/test-and-build-workflow.yml | 2 ++ 5 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/bwc-test-workflow.yml b/.github/workflows/bwc-test-workflow.yml index 51f6c04a8..656aea4cc 100644 --- a/.github/workflows/bwc-test-workflow.yml +++ b/.github/workflows/bwc-test-workflow.yml @@ -17,7 +17,7 @@ jobs: uses: actions/setup-java@v2 with: distribution: temurin # Temurin is a distribution of adoptium - java-version: 17 + java-version: 21 # index-management - name: Checkout Branch uses: actions/checkout@v2 diff --git a/.github/workflows/docker-security-test-workflow.yml b/.github/workflows/docker-security-test-workflow.yml index 951811e57..6511ebfec 100644 --- a/.github/workflows/docker-security-test-workflow.yml +++ b/.github/workflows/docker-security-test-workflow.yml @@ -16,7 +16,7 @@ jobs: uses: actions/setup-java@v2 with: distribution: temurin # Temurin is a distribution of adoptium - java-version: 17 + java-version: 21 - name: Checkout Branch uses: actions/checkout@v2 - name: Build Index Management diff --git a/.github/workflows/multi-node-test-workflow.yml b/.github/workflows/multi-node-test-workflow.yml index b6e762c7e..f23add7cd 100644 --- a/.github/workflows/multi-node-test-workflow.yml +++ b/.github/workflows/multi-node-test-workflow.yml @@ -7,6 +7,8 @@ on: push: branches: - "**" +env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true jobs: Get-CI-Image-Tag: diff --git a/.github/workflows/security-test-workflow.yml b/.github/workflows/security-test-workflow.yml index a4c5aa16f..ddfe712fa 100644 --- a/.github/workflows/security-test-workflow.yml +++ b/.github/workflows/security-test-workflow.yml @@ -7,6 +7,8 @@ on: push: branches: - "**" +env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true jobs: Get-CI-Image-Tag: @@ -31,7 +33,7 @@ jobs: uses: actions/setup-java@v2 with: distribution: temurin # Temurin is a distribution of adoptium - java-version: 17 + java-version: 21 # index-management - name: Checkout Branch uses: actions/checkout@v2 diff --git a/.github/workflows/test-and-build-workflow.yml b/.github/workflows/test-and-build-workflow.yml index 4e8dbf492..73c24da42 100644 --- a/.github/workflows/test-and-build-workflow.yml +++ b/.github/workflows/test-and-build-workflow.yml @@ -6,6 +6,8 @@ on: push: branches: - "**" +env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true jobs: Get-CI-Image-Tag: From 214aae5dec5f23b446b7da8d8c2b154a6b847c27 Mon Sep 17 00:00:00 2001 From: bowenlan-amzn Date: Fri, 19 Jul 2024 16:55:41 -0700 Subject: [PATCH 11/37] Add publish in spi build.gradle (#1207) --- .github/workflows/maven-publish.yml | 3 +- build.gradle | 2 + spi/build.gradle | 58 +++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 1 deletion(-) diff --git a/.github/workflows/maven-publish.yml b/.github/workflows/maven-publish.yml index 8b7b1164b..5e947b345 100644 --- a/.github/workflows/maven-publish.yml +++ b/.github/workflows/maven-publish.yml @@ -32,4 +32,5 @@ jobs: export SONATYPE_PASSWORD=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-password --query SecretString --output text) echo "::add-mask::$SONATYPE_USERNAME" echo "::add-mask::$SONATYPE_PASSWORD" - ./gradlew publishPluginZipPublicationToSnapshotsRepository \ No newline at end of file + ./gradlew publishPluginZipPublicationToSnapshotsRepository + ./gradlew publishShadowPublicationToSnapshotsRepository \ No newline at end of file diff --git a/build.gradle b/build.gradle index 56e49f2ea..6ed9b7091 100644 --- a/build.gradle +++ b/build.gradle @@ -271,6 +271,8 @@ publishing { } } +tasks.generatePomFileForPluginZipPublication.dependsOn publishNebulaPublicationToMavenLocal + plugins.withId('java') { sourceCompatibility = targetCompatibility = JavaVersion.VERSION_11 } diff --git a/spi/build.gradle b/spi/build.gradle index 2fa3f6bf7..3520a964a 100644 --- a/spi/build.gradle +++ b/spi/build.gradle @@ -16,6 +16,7 @@ apply plugin: 'kotlin' apply plugin: 'org.jetbrains.kotlin.jvm' apply plugin: 'org.jetbrains.kotlin.plugin.allopen' apply plugin: 'idea' +apply plugin: 'maven-publish' ext { projectSubstitutions = [:] @@ -83,6 +84,11 @@ tasks.register("sourcesJar", Jar) { from sourceSets.main.allSource } +task javadocJar(type: Jar) { + archiveClassifier = 'javadoc' + from javadoc.destinationDir +} + test { doFirst { test.classpath -= project.files(project.tasks.named('shadowJar')) @@ -103,3 +109,55 @@ check.dependsOn integTest testClusters.javaRestTest { testDistribution = 'INTEG_TEST' } + +publishing { + repositories { + maven { + name = 'staging' + url = "${rootProject.buildDir}/local-staging-repo" + } + maven { + name = "Snapshots" + url = "https://aws.oss.sonatype.org/content/repositories/snapshots" + credentials { + username "$System.env.SONATYPE_USERNAME" + password "$System.env.SONATYPE_PASSWORD" + } + } + } + publications { + shadow(MavenPublication) { + project.shadow.component(it) + + artifact sourcesJar + artifact javadocJar + + pom { + name = "OpenSearch Index Management SPI" + packaging = "jar" + url = "https://github.com/opensearch-project/index-management" + description = "OpenSearch Index Management SPI" + scm { + connection = "scm:git@github.com:opensearch-project/index-management.git" + developerConnection = "scm:git@github.com:opensearch-project/index-management.git" + url = "git@github.com:opensearch-project/index-management.git" + } + licenses { + license { + name = "The Apache License, Version 2.0" + url = "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + } + developers { + developer { + name = "OpenSearch" + url = "https://github.com/opensearch-project/index-management" + } + } + } + } + } + + gradle.startParameter.setShowStacktrace(ShowStacktrace.ALWAYS) + gradle.startParameter.setLogLevel(LogLevel.DEBUG) +} From f02bb01fcdfdec85a2dab368a4045a37da078c76 Mon Sep 17 00:00:00 2001 From: Sarthak Aggarwal Date: Mon, 29 Jul 2024 22:07:40 +0530 Subject: [PATCH 12/37] 2.16 Release Notes (#1214) --- .../opensearch-index-management.release-notes-2.16.0.0.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 release-notes/opensearch-index-management.release-notes-2.16.0.0.md diff --git a/release-notes/opensearch-index-management.release-notes-2.16.0.0.md b/release-notes/opensearch-index-management.release-notes-2.16.0.0.md new file mode 100644 index 000000000..027ad5fb7 --- /dev/null +++ b/release-notes/opensearch-index-management.release-notes-2.16.0.0.md @@ -0,0 +1,8 @@ +## Version 2.16.0.0 2024-07-29 + +Compatible with OpenSearch 2.16.0 + +### Maintenance +* Increment version to 2.16.0-SNAPSHOT ([#1187](https://github.com/opensearch-project/index-management/pull/1187)) +* Add publish in spi build.gradle ([#1207](https://github.com/opensearch-project/index-management/pull/1207)) +* Fix github action ([#1208](https://github.com/opensearch-project/index-management/pull/1208)) From b7ddbb693128e5de31482a0d2bc8c16eef710662 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Fri, 2 Aug 2024 18:16:03 -0400 Subject: [PATCH 13/37] Use adminClient instead of client when interacting with system index in integTests (#1222) --- .../opensearch/indexmanagement/IndexManagementIndicesIT.kt | 2 +- .../opensearch/indexmanagement/IndexManagementRestTestCase.kt | 4 ++-- .../org/opensearch/indexmanagement/SecurityRestTestCase.kt | 2 +- .../notification/filter/NotificationActionListenerIT.kt | 2 +- .../notification/resthandler/LRONConfigRestTestCase.kt | 2 +- .../notification/resthandler/RestIndexLRONConfigActionIT.kt | 2 +- .../indexstatemanagement/IndexStateManagementRestTestCase.kt | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/test/kotlin/org/opensearch/indexmanagement/IndexManagementIndicesIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/IndexManagementIndicesIT.kt index 24a46ace9..21612f19c 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/IndexManagementIndicesIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/IndexManagementIndicesIT.kt @@ -135,7 +135,7 @@ class IndexManagementIndicesIT : IndexStateManagementRestTestCase() { .replace("\"schema_version\": $configSchemaVersion", "\"schema_version\": 0") val entity = StringEntity(mapping, ContentType.APPLICATION_JSON) - client().makeRequest( + adminClient().makeRequest( RestRequest.Method.PUT.toString(), "/$INDEX_MANAGEMENT_INDEX/_mapping", emptyMap(), entity, ) diff --git a/src/test/kotlin/org/opensearch/indexmanagement/IndexManagementRestTestCase.kt b/src/test/kotlin/org/opensearch/indexmanagement/IndexManagementRestTestCase.kt index bb51a1d6b..a55809379 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/IndexManagementRestTestCase.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/IndexManagementRestTestCase.kt @@ -219,7 +219,7 @@ abstract class IndexManagementRestTestCase : ODFERestTestCase() { // Since from the log, this happens very fast (within 0.1~0.2s), the above cluster explain may not have the granularity to catch this. logger.info("Update rollup start time to $startTimeMillis") val response = - client().makeRequest( + adminClient().makeRequest( "POST", "${IndexManagementPlugin.INDEX_MANAGEMENT_INDEX}/_update/${update.id}?wait_for_active_shards=$waitForActiveShards&refresh=true", StringEntity( "{\"doc\":{\"rollup\":{\"schedule\":{\"interval\":{\"start_time\":" + @@ -249,7 +249,7 @@ abstract class IndexManagementRestTestCase : ODFERestTestCase() { val startTimeMillis = desiredStartTimeMillis ?: (Instant.now().toEpochMilli() - millis) val waitForActiveShards = if (isMultiNode) "all" else "1" val response = - client().makeRequest( + adminClient().makeRequest( "POST", "${IndexManagementPlugin.INDEX_MANAGEMENT_INDEX}/_update/${update.id}?wait_for_active_shards=$waitForActiveShards", StringEntity( "{\"doc\":{\"transform\":{\"schedule\":{\"interval\":{\"start_time\":" + diff --git a/src/test/kotlin/org/opensearch/indexmanagement/SecurityRestTestCase.kt b/src/test/kotlin/org/opensearch/indexmanagement/SecurityRestTestCase.kt index d6125a7d8..f6f4231f9 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/SecurityRestTestCase.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/SecurityRestTestCase.kt @@ -492,7 +492,7 @@ abstract class SecurityRestTestCase : IndexManagementRestTestCase() { } protected fun deleteIndexByName(index: String) { - executeRequest(request = Request(RestRequest.Method.DELETE.name, "/$index"), client = client()) + executeRequest(request = Request(RestRequest.Method.DELETE.name, "/$index"), client = adminClient()) } protected fun validateSourceIndex(indexName: String) { diff --git a/src/test/kotlin/org/opensearch/indexmanagement/controlcenter/notification/filter/NotificationActionListenerIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/controlcenter/notification/filter/NotificationActionListenerIT.kt index d95bc26a1..17c855d09 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/controlcenter/notification/filter/NotificationActionListenerIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/controlcenter/notification/filter/NotificationActionListenerIT.kt @@ -401,7 +401,7 @@ class NotificationActionListenerIT : IndexManagementRestTestCase() { closeIndex("source-index") // delete system index - client.makeRequest("DELETE", IndexManagementPlugin.CONTROL_CENTER_INDEX) + adminClient().makeRequest("DELETE", IndexManagementPlugin.CONTROL_CENTER_INDEX) val response = client.makeRequest( diff --git a/src/test/kotlin/org/opensearch/indexmanagement/controlcenter/notification/resthandler/LRONConfigRestTestCase.kt b/src/test/kotlin/org/opensearch/indexmanagement/controlcenter/notification/resthandler/LRONConfigRestTestCase.kt index 00e62a2d1..327e14550 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/controlcenter/notification/resthandler/LRONConfigRestTestCase.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/controlcenter/notification/resthandler/LRONConfigRestTestCase.kt @@ -31,7 +31,7 @@ abstract class LRONConfigRestTestCase : IndexManagementRestTestCase() { @After fun removeAllDocs() { try { - client().makeRequest( + adminClient().makeRequest( "POST", "${IndexManagementPlugin.CONTROL_CENTER_INDEX}/_delete_by_query", mapOf("refresh" to "true"), diff --git a/src/test/kotlin/org/opensearch/indexmanagement/controlcenter/notification/resthandler/RestIndexLRONConfigActionIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/controlcenter/notification/resthandler/RestIndexLRONConfigActionIT.kt index 1e7cdd79a..aa83df837 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/controlcenter/notification/resthandler/RestIndexLRONConfigActionIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/controlcenter/notification/resthandler/RestIndexLRONConfigActionIT.kt @@ -182,7 +182,7 @@ class RestIndexLRONConfigActionIT : LRONConfigRestTestCase() { val lronConfig = randomLRONConfig(taskId = randomTaskId(nodeId = nodeIdsInRestIT.random())) createLRONConfig(lronConfig) - val response = client().makeRequest("GET", "/${IndexManagementPlugin.CONTROL_CENTER_INDEX}/_mapping") + val response = adminClient().makeRequest("GET", "/${IndexManagementPlugin.CONTROL_CENTER_INDEX}/_mapping") val parserMap = createParser(XContentType.JSON.xContent(), response.entity.content).map() as Map> val mappingsMap = parserMap[IndexManagementPlugin.CONTROL_CENTER_INDEX]!!["mappings"] as Map val expected = diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt index 21f27f5af..b2c8d0933 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt @@ -469,7 +469,7 @@ abstract class IndexStateManagementRestTestCase : IndexManagementRestTestCase() val waitForActiveShards = if (isMultiNode) "all" else "1" val endpoint = "$INDEX_MANAGEMENT_INDEX/_update/${update.id}?wait_for_active_shards=$waitForActiveShards;retry_on_conflict=$retryOnConflict" val response = - client().makeRequest( + adminClient().makeRequest( "POST", endpoint, StringEntity( "{\"doc\":{\"managed_index\":{\"schedule\":{\"interval\":{\"start_time\":" + From d6da55cdcb9ac96163644f0bfa295d56db5fd915 Mon Sep 17 00:00:00 2001 From: Sarthak Aggarwal Date: Wed, 21 Aug 2024 09:31:14 +0530 Subject: [PATCH 14/37] skipping execution based on cluster service (#1219) Signed-off-by: Sarthak Aggarwal --- .../indexmanagement/IndexManagementPlugin.kt | 2 +- .../PluginVersionSweepCoordinator.kt | 6 +- .../indexstatemanagement/SkipExecution.kt | 82 ++++++----------- .../coordinator/SkipExecutionTests.kt | 89 +++++++++++++++---- 4 files changed, 103 insertions(+), 76 deletions(-) diff --git a/src/main/kotlin/org/opensearch/indexmanagement/IndexManagementPlugin.kt b/src/main/kotlin/org/opensearch/indexmanagement/IndexManagementPlugin.kt index 7660d48e1..e59602205 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/IndexManagementPlugin.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/IndexManagementPlugin.kt @@ -411,7 +411,7 @@ class IndexManagementPlugin : JobSchedulerExtension, NetworkPlugin, ActionPlugin fieldCapsFilter = FieldCapsFilter(clusterService, settings, indexNameExpressionResolver) this.indexNameExpressionResolver = indexNameExpressionResolver - val skipFlag = SkipExecution(client) + val skipFlag = SkipExecution() RollupFieldValueExpressionResolver.registerServices(scriptService, clusterService) val rollupRunner = RollupRunner diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/PluginVersionSweepCoordinator.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/PluginVersionSweepCoordinator.kt index 20dc3bcea..55c7511ba 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/PluginVersionSweepCoordinator.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/PluginVersionSweepCoordinator.kt @@ -25,7 +25,7 @@ class PluginVersionSweepCoordinator( private val skipExecution: SkipExecution, settings: Settings, private val threadPool: ThreadPool, - clusterService: ClusterService, + var clusterService: ClusterService, ) : CoroutineScope by CoroutineScope(SupervisorJob() + Dispatchers.Default + CoroutineName("ISMPluginSweepCoordinator")), LifecycleListener(), ClusterStateListener { @@ -58,7 +58,7 @@ class PluginVersionSweepCoordinator( override fun clusterChanged(event: ClusterChangedEvent) { if (event.nodesChanged() || event.isNewCluster) { - skipExecution.sweepISMPluginVersion() + skipExecution.sweepISMPluginVersion(clusterService) initBackgroundSweepISMPluginVersionExecution() } } @@ -77,7 +77,7 @@ class PluginVersionSweepCoordinator( logger.info("Canceling sweep ism plugin version job") scheduledSkipExecution?.cancel() } else { - skipExecution.sweepISMPluginVersion() + skipExecution.sweepISMPluginVersion(clusterService) } } catch (e: Exception) { logger.error("Failed to sweep ism plugin version", e) diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/SkipExecution.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/SkipExecution.kt index c74f27c89..1d0d6c841 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/SkipExecution.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/SkipExecution.kt @@ -6,20 +6,14 @@ package org.opensearch.indexmanagement.indexstatemanagement import org.apache.logging.log4j.LogManager -import org.opensearch.action.admin.cluster.node.info.NodesInfoAction -import org.opensearch.action.admin.cluster.node.info.NodesInfoRequest -import org.opensearch.action.admin.cluster.node.info.NodesInfoResponse -import org.opensearch.action.admin.cluster.node.info.PluginsAndModules -import org.opensearch.client.Client -import org.opensearch.core.action.ActionListener +import org.opensearch.Version +import org.opensearch.cluster.service.ClusterService import org.opensearch.indexmanagement.util.OpenForTesting // TODO this can be moved to job scheduler, so that all extended plugin // can avoid running jobs in an upgrading cluster @OpenForTesting -class SkipExecution( - private val client: Client, -) { +class SkipExecution { private val logger = LogManager.getLogger(javaClass) @Volatile @@ -31,53 +25,27 @@ class SkipExecution( final var hasLegacyPlugin: Boolean = false private set - fun sweepISMPluginVersion() { - // if old version ISM plugin exists (2 versions ISM in one cluster), set skip flag to true - val request = NodesInfoRequest().clear().addMetric("plugins") - client.execute( - NodesInfoAction.INSTANCE, request, - object : ActionListener { - override fun onResponse(response: NodesInfoResponse) { - val versionSet = mutableSetOf() - val legacyVersionSet = mutableSetOf() - - response.nodes.map { it.getInfo(PluginsAndModules::class.java).pluginInfos } - .forEach { - it.forEach { nodePlugin -> - if (nodePlugin.name == "opensearch-index-management" || - nodePlugin.name == "opensearch_index_management" - ) { - versionSet.add(nodePlugin.version) - } - - if (nodePlugin.name == "opendistro-index-management" || - nodePlugin.name == "opendistro_index_management" - ) { - legacyVersionSet.add(nodePlugin.version) - } - } - } - - if ((versionSet.size + legacyVersionSet.size) > 1) { - flag = true - logger.info("There are multiple versions of Index Management plugins in the cluster: [$versionSet, $legacyVersionSet]") - } else { - flag = false - } - - if (versionSet.isNotEmpty() && legacyVersionSet.isNotEmpty()) { - hasLegacyPlugin = true - logger.info("Found legacy plugin versions [$legacyVersionSet] and opensearch plugins versions [$versionSet] in the cluster") - } else { - hasLegacyPlugin = false - } - } - - override fun onFailure(e: Exception) { - logger.error("Failed sweeping nodes for ISM plugin versions: $e") - flag = false - } - }, - ) + fun sweepISMPluginVersion(clusterService: ClusterService) { + try { + // if old version ISM plugin exists (2 versions ISM in one cluster), set skip flag to true + val currentMinVersion = clusterService.state().nodes.minNodeVersion + val currentMaxVersion = clusterService.state().nodes.maxNodeVersion + + if (currentMinVersion != null && !currentMinVersion.equals(currentMaxVersion)) { + flag = true + logger.info("There are multiple versions of Index Management plugins in the cluster: [$currentMaxVersion, $currentMinVersion]") + } else { + flag = false + } + + if (currentMinVersion.major > Version.CURRENT.major && currentMinVersion != currentMaxVersion) { + hasLegacyPlugin = true + logger.info("Found legacy plugin versions [$currentMinVersion] and opensearch plugins versions [$currentMaxVersion] in the cluster") + } else { + hasLegacyPlugin = false + } + } catch (e: Exception) { + logger.error("Unable to fetch node versions from cluster service", e) + } } } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/coordinator/SkipExecutionTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/coordinator/SkipExecutionTests.kt index 2421ff5f1..d41de106a 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/coordinator/SkipExecutionTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/coordinator/SkipExecutionTests.kt @@ -5,29 +5,88 @@ package org.opensearch.indexmanagement.indexstatemanagement.coordinator +import com.nhaarman.mockitokotlin2.mock +import com.nhaarman.mockitokotlin2.whenever import org.junit.Before -import org.mockito.Mockito -import org.opensearch.action.admin.cluster.node.info.NodesInfoAction -import org.opensearch.client.Client -import org.opensearch.cluster.ClusterChangedEvent -import org.opensearch.cluster.OpenSearchAllocationTestCase +import org.opensearch.Version +import org.opensearch.cluster.ClusterState +import org.opensearch.cluster.node.DiscoveryNode +import org.opensearch.cluster.node.DiscoveryNodes +import org.opensearch.cluster.service.ClusterService +import org.opensearch.core.common.transport.TransportAddress import org.opensearch.indexmanagement.indexstatemanagement.SkipExecution +import org.opensearch.test.OpenSearchTestCase -class SkipExecutionTests : OpenSearchAllocationTestCase() { - private lateinit var client: Client +class SkipExecutionTests : OpenSearchTestCase() { + private var clusterService: ClusterService = mock() + private lateinit var clusterState: ClusterState private lateinit var skip: SkipExecution @Before - @Throws(Exception::class) fun setup() { - client = Mockito.mock(Client::class.java) - skip = SkipExecution(client) + skip = SkipExecution() } - fun `test cluster change event`() { - val event = Mockito.mock(ClusterChangedEvent::class.java) - Mockito.`when`(event.nodesChanged()).thenReturn(true) - skip.sweepISMPluginVersion() - Mockito.verify(client).execute(Mockito.eq(NodesInfoAction.INSTANCE), Mockito.any(), Mockito.any()) + fun `test sweepISMPluginVersion should set flag to false and hasLegacyPlugin to false when all nodes have the same version`() { + val version = Version.CURRENT + val node1 = DiscoveryNode("node1", TransportAddress(TransportAddress.META_ADDRESS, 9300), version) + val node2 = DiscoveryNode("node2", TransportAddress(TransportAddress.META_ADDRESS, 9301), version) + val discoveryNodes = DiscoveryNodes.builder().add(node1).add(node2).build() + clusterState = ClusterState.builder(ClusterState.EMPTY_STATE).nodes(discoveryNodes).build() + whenever(clusterService.state()).thenReturn(clusterState) + + skip.sweepISMPluginVersion(clusterService) + + assertFalse(skip.flag) + assertFalse(skip.hasLegacyPlugin) + } + + fun `test sweepISMPluginVersion should set flag to true and hasLegacyPlugin to false when all nodes have the different versions`() { + val version1 = Version.CURRENT + val version2 = Version.V_2_0_0 + val node1 = DiscoveryNode("node1", TransportAddress(TransportAddress.META_ADDRESS, 9300), version1) + val node2 = DiscoveryNode("node2", TransportAddress(TransportAddress.META_ADDRESS, 9301), version2) + val node3 = DiscoveryNode("node3", TransportAddress(TransportAddress.META_ADDRESS, 9302), version2) + val discoveryNodes = DiscoveryNodes.builder().add(node1).add(node2).add(node3).build() + clusterState = ClusterState.builder(ClusterState.EMPTY_STATE).nodes(discoveryNodes).build() + whenever(clusterService.state()).thenReturn(clusterState) + + skip.sweepISMPluginVersion(clusterService) + + assertTrue(skip.flag) + assertFalse(skip.hasLegacyPlugin) + } + + fun `test sweepISMPluginVersion should set flag to true and hasLegacyPlugin to true when there are different versions including current version`() { + val minVersion = Version.fromString("7.10.0") + val maxVersion = Version.CURRENT + val node1 = DiscoveryNode("node1", TransportAddress(TransportAddress.META_ADDRESS, 9300), minVersion) + val node2 = DiscoveryNode("node2", TransportAddress(TransportAddress.META_ADDRESS, 9301), maxVersion) + val node3 = DiscoveryNode("node3", TransportAddress(TransportAddress.META_ADDRESS, 9302), maxVersion) + val discoveryNodes = DiscoveryNodes.builder().add(node1).add(node2).add(node3).build() + clusterState = ClusterState.builder(ClusterState.EMPTY_STATE).nodes(discoveryNodes).build() + whenever(clusterService.state()).thenReturn(clusterState) + + skip.sweepISMPluginVersion(clusterService) + + assertTrue(skip.flag) + assertTrue(skip.hasLegacyPlugin) + } + + fun `test sweepISMPluginVersion should set flag to true and hasLegacyPlugin to true with different versions`() { + val minVersion = Version.fromString("7.10.0") + val maxVersion = Version.V_2_0_0 + val node1 = DiscoveryNode("node1", TransportAddress(TransportAddress.META_ADDRESS, 9300), minVersion) + val node2 = DiscoveryNode("node2", TransportAddress(TransportAddress.META_ADDRESS, 9301), maxVersion) + val node3 = DiscoveryNode("node3", TransportAddress(TransportAddress.META_ADDRESS, 9302), maxVersion) + + val discoveryNodes = DiscoveryNodes.builder().add(node1).add(node2).add(node3).build() + clusterState = ClusterState.builder(ClusterState.EMPTY_STATE).nodes(discoveryNodes).build() + whenever(clusterService.state()).thenReturn(clusterState) + + skip.sweepISMPluginVersion(clusterService) + + assertTrue(skip.flag) + assertTrue(skip.hasLegacyPlugin) } } From 4f3a7d85efe1459822300028a78d476d0560b587 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Mon, 26 Aug 2024 00:43:25 -0400 Subject: [PATCH 15/37] Move non-active maintainers to emeritus. (#1233) Signed-off-by: dblock --- .github/CODEOWNERS | 2 +- MAINTAINERS.md | 21 +++++++++++++-------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 07d643136..67a22979a 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -* @bowenlan-amzn @getsaurabh02 @lezzago @praveensameneni @xluo-aws @gaobinlong @Hailong-am @amsiglan @sbcd90 @eirsep @AWSHurneyt @jowg-amazon @r1walz @vikasvb90 +* @bowenlan-amzn @Hailong-am @r1walz @vikasvb90 diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 39016b7a7..60cc5d0f4 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -6,17 +6,22 @@ This document contains a list of maintainers in this repo. See [opensearch-proje | Maintainer | GitHub ID | Affiliation | |-----------------------|-------------------------------------------------------| ----------- | -| Ashish Agrawal | [lezzago](https://github.com/lezzago) | Amazon | +| Vikas Bansal | [vikasvb90](https://github.com/vikasvb90) | Amazon | | Bowen Lan | [bowenlan-amzn](https://github.com/bowenlan-amzn) | Amazon | -| Praveen Sameneni | [praveensameneni](https://github.com/praveensameneni) | Amazon | -| Saurabh Singh | [getsaurabh02](https://github.com/getsaurabh02/) | Amazon | -| Xuesong Luo | [xluo-aws](https://github.com/xluo-aws) | Amazon | | Hailong Cui | [Hailong-am](https://github.com/Hailong-am) | Amazon | -| Binlong Gao | [gaobinlong](https://github.com/gaobinlong) | Amazon | +| Rohit Ashiwal | [r1walz](https://github.com/r1walz) | Independent | + +## Emeritus + +| Maintainer | GitHub ID | Affiliation | +|-----------------------|-------------------------------------------------------| ----------- | | Amardeepsingh Siglani | [amsiglan](https://github.com/amsiglan) | Amazon | +| Binlong Gao | [gaobinlong](https://github.com/gaobinlong) | Amazon | +| Ashish Agrawal | [lezzago](https://github.com/lezzago) | Amazon | +| Joanne Wang | [jowg-amazon](https://github.com/jowg-amazon) | Amazon | +| Praveen Sameneni | [praveensameneni](https://github.com/praveensameneni) | Amazon | +| Saurabh Singh | [getsaurabh02](https://github.com/getsaurabh02/) | Amazon | | Subhobrata Dey | [sbcd90](https://github.com/sbcd90) | Amazon | | Surya Sashank Nistala | [eirsep](https://github.com/eirsep) | Amazon | | Thomas Hurney | [AWSHurneyt](https://github.com/AWSHurneyt) | Amazon | -| Joanne Wang | [jowg-amazon](https://github.com/jowg-amazon) | Amazon | -| Rohit Ashiwal | [r1walz](https://github.com/r1walz) | Amazon | -| Vikas Bansal | [vikasvb90](https://github.com/vikasvb90) | Amazon | +| Xuesong Luo | [xluo-aws](https://github.com/xluo-aws) | Amazon | From c0942eb60921070385d1b10c7453eee5e148e44f Mon Sep 17 00:00:00 2001 From: Kshitij Tandon Date: Thu, 5 Sep 2024 18:55:27 +0530 Subject: [PATCH 16/37] Added release notes for v2.17 (#1238) --- ...nsearch-index-management.release-notes-2.17.0.0.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 release-notes/opensearch-index-management.release-notes-2.17.0.0.md diff --git a/release-notes/opensearch-index-management.release-notes-2.17.0.0.md b/release-notes/opensearch-index-management.release-notes-2.17.0.0.md new file mode 100644 index 000000000..e33508f4c --- /dev/null +++ b/release-notes/opensearch-index-management.release-notes-2.17.0.0.md @@ -0,0 +1,11 @@ +## Version 2.17.0.0 2024-09-05 + +Compatible with OpenSearch 2.17.0 + +### Bug fixes +* Skipping execution based on cluster service ([#1219](https://github.com/opensearch-project/index-management/pull/1219)) + +### Maintenance +* Increment version to 2.17.0-SNAPSHOT ([#1221](https://github.com/opensearch-project/index-management/pull/1221)) +* Use adminClient instead of client when interacting with system index in integTests ([#1222](https://github.com/opensearch-project/index-management/pull/1222)) +* Move non-active maintainers to emeritus ([#1233](https://github.com/opensearch-project/index-management/pull/1233)) \ No newline at end of file From 395317dcc40762fc05b4320885c2ea27be572324 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Tue, 10 Sep 2024 21:29:22 -0400 Subject: [PATCH 17/37] Update CI check for integ-test-with-security to run all integ tests with security (#1243) * Run all integ tests with security Signed-off-by: Craig Perkins * Use adminClient when deleting ISM index in RestStopRollupActionIT Signed-off-by: Craig Perkins * Update RolloverActionIT Signed-off-by: Craig Perkins * Fix RestStopSnapshotManagementIT Signed-off-by: Craig Perkins * Fix RestChangePolicyActionIT Signed-off-by: Craig Perkins * Fix more tests Signed-off-by: Craig Perkins * Fix more tests Signed-off-by: Craig Perkins * Fix more tests Signed-off-by: Craig Perkins * Fix RestExplainTransformActionIT Signed-off-by: Craig Perkins * Fix more tests Signed-off-by: Craig Perkins * Make sure read from system index also uses adminClient Signed-off-by: Craig Perkins --------- Signed-off-by: Craig Perkins --- .github/workflows/security-test-workflow.yml | 2 +- .../IndexStateManagementRestTestCase.kt | 4 ++-- .../indexstatemanagement/action/RolloverActionIT.kt | 2 +- .../resthandler/RestChangePolicyActionIT.kt | 8 ++++++-- .../rollup/resthandler/RestStartRollupActionIT.kt | 9 ++++++--- .../rollup/resthandler/RestStopRollupActionIT.kt | 7 +++++-- .../indexmanagement/rollup/runner/RollupRunnerIT.kt | 2 +- .../snapshotmanagement/SnapshotManagementRestTestCase.kt | 4 ++-- .../resthandler/RestDeleteSnapshotManagementIT.kt | 4 +++- .../resthandler/RestExplainSnapshotManagementIT.kt | 4 +++- .../resthandler/RestGetSnapshotManagementIT.kt | 5 ++++- .../resthandler/RestIndexSnapshotManagementIT.kt | 4 +++- .../resthandler/RestStartSnapshotManagementIT.kt | 5 ++++- .../resthandler/RestStopSnapshotManagementIT.kt | 5 ++++- .../indexmanagement/transform/TransformRestTestCase.kt | 2 +- .../transform/resthandler/RestDeleteTransformActionIT.kt | 6 +++++- .../resthandler/RestExplainTransformActionIT.kt | 6 ++++-- 17 files changed, 55 insertions(+), 24 deletions(-) diff --git a/.github/workflows/security-test-workflow.yml b/.github/workflows/security-test-workflow.yml index ddfe712fa..d88750bda 100644 --- a/.github/workflows/security-test-workflow.yml +++ b/.github/workflows/security-test-workflow.yml @@ -40,7 +40,7 @@ jobs: - name: Run integration tests run: | chown -R 1000:1000 `pwd` - su `id -un 1000` -c "./gradlew integTest -Dsecurity=true -Dhttps=true --tests '*SecurityBehaviorIT'" + su `id -un 1000` -c "./gradlew integTest -Dsecurity=true -Dhttps=true --tests '*IT'" - name: Upload failed logs uses: actions/upload-artifact@v2 if: failure() diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt index b2c8d0933..1e37cdf4b 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt @@ -351,7 +351,7 @@ abstract class IndexStateManagementRestTestCase : IndexManagementRestTestCase() } """.trimIndent() val res = - client().makeRequest( + adminClient().makeRequest( "PUT", "$index/_settings", emptyMap(), StringEntity(body, ContentType.APPLICATION_JSON), ) @@ -483,7 +483,7 @@ abstract class IndexStateManagementRestTestCase : IndexManagementRestTestCase() protected fun updateManagedIndexConfigPolicySeqNo(update: ManagedIndexConfig) { val response = - client().makeRequest( + adminClient().makeRequest( "POST", "$INDEX_MANAGEMENT_INDEX/_update/${update.id}", StringEntity( "{\"doc\":{\"managed_index\":{\"policy_seq_no\":\"${update.policySeqNo}\"}}}", diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/RolloverActionIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/RolloverActionIT.kt index a45120b6c..c12a77cba 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/RolloverActionIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/RolloverActionIT.kt @@ -784,7 +784,7 @@ class RolloverActionIT : IndexStateManagementRestTestCase() { } // Manually produce transaction failure val response = - client().makeRequest( + adminClient().makeRequest( "POST", "$INDEX_MANAGEMENT_INDEX/_update/${managedIndexConfig.id}%23metadata", StringEntity( "{\n" + diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestChangePolicyActionIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestChangePolicyActionIT.kt index 463b745d4..73e04f928 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestChangePolicyActionIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestChangePolicyActionIT.kt @@ -6,6 +6,7 @@ package org.opensearch.indexmanagement.indexstatemanagement.resthandler import org.junit.Before +import org.opensearch.client.Request import org.opensearch.client.ResponseException import org.opensearch.common.settings.Settings import org.opensearch.core.rest.RestStatus @@ -35,7 +36,7 @@ import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StateMetaDa import org.opensearch.indexmanagement.waitFor import org.opensearch.rest.RestRequest import java.time.Instant -import java.util.Locale +import java.util.* class RestChangePolicyActionIT : IndexStateManagementRestTestCase() { private val testIndexName = javaClass.simpleName.lowercase(Locale.ROOT) @@ -89,7 +90,10 @@ class RestChangePolicyActionIT : IndexStateManagementRestTestCase() { } fun `test nonexistent ism config index`() { - if (indexExists(INDEX_MANAGEMENT_INDEX)) deleteIndex(INDEX_MANAGEMENT_INDEX) + if (indexExists(INDEX_MANAGEMENT_INDEX)) { + val deleteISMIndexRequest = Request("DELETE", "/$INDEX_MANAGEMENT_INDEX") + adminClient().performRequest(deleteISMIndexRequest) + } try { val changePolicy = ChangePolicy("some_id", null, emptyList(), false) client().makeRequest( diff --git a/src/test/kotlin/org/opensearch/indexmanagement/rollup/resthandler/RestStartRollupActionIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/rollup/resthandler/RestStartRollupActionIT.kt index 45a82ef39..6baf8b474 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/rollup/resthandler/RestStartRollupActionIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/rollup/resthandler/RestStartRollupActionIT.kt @@ -5,11 +5,13 @@ package org.opensearch.indexmanagement.rollup.resthandler +import org.opensearch.client.Request import org.opensearch.client.ResponseException import org.opensearch.common.settings.Settings import org.opensearch.core.rest.RestStatus import org.opensearch.indexmanagement.IndexManagementIndices import org.opensearch.indexmanagement.IndexManagementPlugin +import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.ROLLUP_JOBS_BASE_URI import org.opensearch.indexmanagement.common.model.dimension.DateHistogram import org.opensearch.indexmanagement.indexstatemanagement.util.INDEX_HIDDEN @@ -22,7 +24,7 @@ import org.opensearch.indexmanagement.waitFor import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule import java.time.Instant import java.time.temporal.ChronoUnit -import java.util.Locale +import java.util.* class RestStartRollupActionIT : RollupRestAPITestCase() { private val testName = javaClass.simpleName.lowercase(Locale.ROOT) @@ -200,7 +202,8 @@ class RestStartRollupActionIT : RollupRestAPITestCase() { fun `test start rollup when multiple shards configured for IM config index`() { // setup ism-config index with multiple primary shards - deleteIndex(IndexManagementPlugin.INDEX_MANAGEMENT_INDEX) + val deleteISMIndexRequest = Request("DELETE", "/$INDEX_MANAGEMENT_INDEX") + adminClient().performRequest(deleteISMIndexRequest) val mapping = IndexManagementIndices.indexManagementMappings.trim().trimStart('{').trimEnd('}') val settings = Settings.builder() @@ -253,7 +256,7 @@ class RestStartRollupActionIT : RollupRestAPITestCase() { // clearing the config index to prevent other tests using this multi shard index Thread.sleep(2000L) - deleteIndex(IndexManagementPlugin.INDEX_MANAGEMENT_INDEX) + adminClient().performRequest(deleteISMIndexRequest) Thread.sleep(2000L) } } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/rollup/resthandler/RestStopRollupActionIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/rollup/resthandler/RestStopRollupActionIT.kt index d4eada87d..4789e1f50 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/rollup/resthandler/RestStopRollupActionIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/rollup/resthandler/RestStopRollupActionIT.kt @@ -5,11 +5,13 @@ package org.opensearch.indexmanagement.rollup.resthandler +import org.opensearch.client.Request import org.opensearch.client.ResponseException import org.opensearch.common.settings.Settings import org.opensearch.core.rest.RestStatus import org.opensearch.indexmanagement.IndexManagementIndices import org.opensearch.indexmanagement.IndexManagementPlugin +import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.ROLLUP_JOBS_BASE_URI import org.opensearch.indexmanagement.common.model.dimension.DateHistogram import org.opensearch.indexmanagement.common.model.dimension.Terms @@ -256,7 +258,8 @@ class RestStopRollupActionIT : RollupRestAPITestCase() { fun `test stop rollup when multiple shards configured for IM config index`() { // setup ism-config index with multiple primary shards - deleteIndex(IndexManagementPlugin.INDEX_MANAGEMENT_INDEX) + val deleteISMIndexRequest = Request("DELETE", "/$INDEX_MANAGEMENT_INDEX") + adminClient().performRequest(deleteISMIndexRequest) val mapping = IndexManagementIndices.indexManagementMappings.trim().trimStart('{').trimEnd('}') val settings = Settings.builder() @@ -313,7 +316,7 @@ class RestStopRollupActionIT : RollupRestAPITestCase() { // clearing the config index to prevent other tests using this multi shard index Thread.sleep(2000L) - deleteIndex(IndexManagementPlugin.INDEX_MANAGEMENT_INDEX) + adminClient().performRequest(deleteISMIndexRequest) Thread.sleep(2000L) } } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/rollup/runner/RollupRunnerIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/rollup/runner/RollupRunnerIT.kt index 09dedecb8..e085f6c30 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/rollup/runner/RollupRunnerIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/rollup/runner/RollupRunnerIT.kt @@ -1466,7 +1466,7 @@ class RollupRunnerIT : RollupRestTestCase() { // - Source index with pattern mapping to some closed indices private fun deleteRollupMetadata(metadataId: String) { - val response = client().makeRequest("DELETE", "${IndexManagementPlugin.INDEX_MANAGEMENT_INDEX}/_doc/$metadataId") + val response = adminClient().makeRequest("DELETE", "${IndexManagementPlugin.INDEX_MANAGEMENT_INDEX}/_doc/$metadataId") assertEquals("Unable to delete rollup metadata $metadataId", RestStatus.OK, response.restStatus()) } } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/SnapshotManagementRestTestCase.kt b/src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/SnapshotManagementRestTestCase.kt index 5c98260fa..d4078288e 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/SnapshotManagementRestTestCase.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/SnapshotManagementRestTestCase.kt @@ -139,7 +139,7 @@ abstract class SnapshotManagementRestTestCase : IndexManagementRestTestCase() { val startTimeMillis = desiredStartTimeMillis ?: (now().toEpochMilli() - millis) val waitForActiveShards = if (isMultiNode) "all" else "1" val response = - client().makeRequest( + adminClient().makeRequest( "POST", "$INDEX_MANAGEMENT_INDEX/_update/${update.id}?wait_for_active_shards=$waitForActiveShards", StringEntity( "{\"doc\":{\"sm_policy\":{\"schedule\":{\"interval\":{\"start_time\":\"$startTimeMillis\"}}}}}", @@ -171,7 +171,7 @@ abstract class SnapshotManagementRestTestCase : IndexManagementRestTestCase() { val startTimeMillis = desiredStartTimeMillis ?: (now().toEpochMilli() - millis) val waitForActiveShards = if (isMultiNode) "all" else "1" val response = - client().makeRequest( + adminClient().makeRequest( "POST", "$INDEX_MANAGEMENT_INDEX/_update/${update.metadataID}?wait_for_active_shards=$waitForActiveShards", StringEntity( """ diff --git a/src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/resthandler/RestDeleteSnapshotManagementIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/resthandler/RestDeleteSnapshotManagementIT.kt index 7464c0e64..13ecd4a71 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/resthandler/RestDeleteSnapshotManagementIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/resthandler/RestDeleteSnapshotManagementIT.kt @@ -5,6 +5,7 @@ package org.opensearch.indexmanagement.snapshotmanagement.resthandler +import org.opensearch.client.Request import org.opensearch.client.ResponseException import org.opensearch.core.rest.RestStatus import org.opensearch.indexmanagement.IndexManagementPlugin @@ -38,7 +39,8 @@ class RestDeleteSnapshotManagementIT : SnapshotManagementRestTestCase() { fun `test deleting a snapshot management policy that doesn't exist and config index doesnt exist`() { try { - deleteIndex(INDEX_MANAGEMENT_INDEX) + val deleteISMIndexRequest = Request("DELETE", "/$INDEX_MANAGEMENT_INDEX") + adminClient().performRequest(deleteISMIndexRequest) client().makeRequest("DELETE", "${IndexManagementPlugin.SM_POLICIES_URI}/nonexistent_policy") fail("expected 404 ResponseException") } catch (e: ResponseException) { diff --git a/src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/resthandler/RestExplainSnapshotManagementIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/resthandler/RestExplainSnapshotManagementIT.kt index 1e8d19848..ce64e240f 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/resthandler/RestExplainSnapshotManagementIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/resthandler/RestExplainSnapshotManagementIT.kt @@ -5,6 +5,7 @@ package org.opensearch.indexmanagement.snapshotmanagement.resthandler +import org.opensearch.client.Request import org.opensearch.client.ResponseException import org.opensearch.common.xcontent.XContentType import org.opensearch.core.rest.RestStatus @@ -141,7 +142,8 @@ class RestExplainSnapshotManagementIT : SnapshotManagementRestTestCase() { fun `test explain sm policy when config index doesn't exist`() { try { - deleteIndex(INDEX_MANAGEMENT_INDEX) + val deleteISMIndexRequest = Request("DELETE", "/$INDEX_MANAGEMENT_INDEX") + adminClient().performRequest(deleteISMIndexRequest) explainSMPolicy(randomAlphaOfLength(10)) fail("expected response exception") } catch (e: ResponseException) { diff --git a/src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/resthandler/RestGetSnapshotManagementIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/resthandler/RestGetSnapshotManagementIT.kt index d65e2a5fa..ee9eba36c 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/resthandler/RestGetSnapshotManagementIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/resthandler/RestGetSnapshotManagementIT.kt @@ -7,9 +7,11 @@ package org.opensearch.indexmanagement.snapshotmanagement.resthandler import org.apache.hc.core5.http.HttpHeaders import org.apache.hc.core5.http.message.BasicHeader +import org.opensearch.client.Request import org.opensearch.client.ResponseException import org.opensearch.core.rest.RestStatus import org.opensearch.indexmanagement.IndexManagementPlugin +import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX import org.opensearch.indexmanagement.makeRequest import org.opensearch.indexmanagement.opensearchapi.convertToMap import org.opensearch.indexmanagement.snapshotmanagement.SnapshotManagementRestTestCase @@ -48,7 +50,8 @@ class RestGetSnapshotManagementIT : SnapshotManagementRestTestCase() { @Throws(Exception::class) fun `test getting a snapshot management policy that doesn't exist and config index doesnt exist`() { try { - deleteIndex(IndexManagementPlugin.INDEX_MANAGEMENT_INDEX) + val deleteISMIndexRequest = Request("DELETE", "/$INDEX_MANAGEMENT_INDEX") + adminClient().performRequest(deleteISMIndexRequest) getSMPolicy(randomAlphaOfLength(20)) fail("expected response exception") } catch (e: ResponseException) { diff --git a/src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/resthandler/RestIndexSnapshotManagementIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/resthandler/RestIndexSnapshotManagementIT.kt index 6a7975dd0..f40467f81 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/resthandler/RestIndexSnapshotManagementIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/resthandler/RestIndexSnapshotManagementIT.kt @@ -5,6 +5,7 @@ package org.opensearch.indexmanagement.snapshotmanagement.resthandler +import org.opensearch.client.Request import org.opensearch.client.ResponseException import org.opensearch.common.xcontent.XContentType import org.opensearch.core.rest.RestStatus @@ -130,7 +131,8 @@ class RestIndexSnapshotManagementIT : SnapshotManagementRestTestCase() { @Throws(Exception::class) @Suppress("UNCHECKED_CAST") fun `test mappings after sm policy creation`() { - deleteIndex(INDEX_MANAGEMENT_INDEX) + val deleteISMIndexRequest = Request("DELETE", "/$INDEX_MANAGEMENT_INDEX") + adminClient().performRequest(deleteISMIndexRequest) createSMPolicy(randomSMPolicy()) val response = client().makeRequest("GET", "/$INDEX_MANAGEMENT_INDEX/_mapping") diff --git a/src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/resthandler/RestStartSnapshotManagementIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/resthandler/RestStartSnapshotManagementIT.kt index 4ec67011a..8f8d0c67c 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/resthandler/RestStartSnapshotManagementIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/resthandler/RestStartSnapshotManagementIT.kt @@ -5,9 +5,11 @@ package org.opensearch.indexmanagement.snapshotmanagement.resthandler +import org.opensearch.client.Request import org.opensearch.client.ResponseException import org.opensearch.core.rest.RestStatus import org.opensearch.indexmanagement.IndexManagementPlugin +import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX import org.opensearch.indexmanagement.makeRequest import org.opensearch.indexmanagement.snapshotmanagement.SnapshotManagementRestTestCase import org.opensearch.indexmanagement.snapshotmanagement.randomSMPolicy @@ -59,7 +61,8 @@ class RestStartSnapshotManagementIT : SnapshotManagementRestTestCase() { fun `test starting a snapshot management policy with no config index fails`() { try { - deleteIndex(IndexManagementPlugin.INDEX_MANAGEMENT_INDEX) + val request = Request("DELETE", "/$INDEX_MANAGEMENT_INDEX") + adminClient().performRequest(request) client().makeRequest("POST", "${IndexManagementPlugin.SM_POLICIES_URI}/nonexistent_foo/_start") fail("expected response exception") } catch (e: ResponseException) { diff --git a/src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/resthandler/RestStopSnapshotManagementIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/resthandler/RestStopSnapshotManagementIT.kt index 77569632f..fa308e552 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/resthandler/RestStopSnapshotManagementIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/resthandler/RestStopSnapshotManagementIT.kt @@ -5,9 +5,11 @@ package org.opensearch.indexmanagement.snapshotmanagement.resthandler +import org.opensearch.client.Request import org.opensearch.client.ResponseException import org.opensearch.core.rest.RestStatus import org.opensearch.indexmanagement.IndexManagementPlugin +import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX import org.opensearch.indexmanagement.makeRequest import org.opensearch.indexmanagement.snapshotmanagement.SnapshotManagementRestTestCase import org.opensearch.indexmanagement.snapshotmanagement.randomSMPolicy @@ -59,7 +61,8 @@ class RestStopSnapshotManagementIT : SnapshotManagementRestTestCase() { fun `test stopping a snapshot management policy with no config index fails`() { try { - deleteIndex(IndexManagementPlugin.INDEX_MANAGEMENT_INDEX) + val deleteISMIndexRequest = Request("DELETE", "/$INDEX_MANAGEMENT_INDEX") + adminClient().performRequest(deleteISMIndexRequest) client().makeRequest("POST", "${IndexManagementPlugin.SM_POLICIES_URI}/nonexistent_foo/_stop") fail("expected response exception") } catch (e: ResponseException) { diff --git a/src/test/kotlin/org/opensearch/indexmanagement/transform/TransformRestTestCase.kt b/src/test/kotlin/org/opensearch/indexmanagement/transform/TransformRestTestCase.kt index e62f7321d..557ecc47b 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/transform/TransformRestTestCase.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/transform/TransformRestTestCase.kt @@ -119,7 +119,7 @@ abstract class TransformRestTestCase : IndexManagementRestTestCase() { protected fun getTransformMetadata(metadataId: String): TransformMetadata { val response = - client().makeRequest( + adminClient().makeRequest( "GET", "$INDEX_MANAGEMENT_INDEX/_doc/$metadataId", null, BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json"), ) assertEquals("Unable to get transform metadata $metadataId", RestStatus.OK, response.restStatus()) diff --git a/src/test/kotlin/org/opensearch/indexmanagement/transform/resthandler/RestDeleteTransformActionIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/transform/resthandler/RestDeleteTransformActionIT.kt index 0aa807970..bc58daa3e 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/transform/resthandler/RestDeleteTransformActionIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/transform/resthandler/RestDeleteTransformActionIT.kt @@ -5,6 +5,7 @@ package org.opensearch.indexmanagement.transform.resthandler +import org.opensearch.client.Request import org.opensearch.client.ResponseException import org.opensearch.core.rest.RestStatus import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX @@ -76,7 +77,10 @@ class RestDeleteTransformActionIT : TransformRestTestCase() { @Throws(Exception::class) fun `test deleting a transform that doesn't exist and config index doesn't exist`() { try { - if (indexExists(INDEX_MANAGEMENT_INDEX)) deleteIndex(INDEX_MANAGEMENT_INDEX) + if (indexExists(INDEX_MANAGEMENT_INDEX)) { + val deleteISMIndexRequest = Request("DELETE", "/$INDEX_MANAGEMENT_INDEX") + adminClient().performRequest(deleteISMIndexRequest) + } val res = client().makeRequest("DELETE", "$TRANSFORM_BASE_URI/foobarbaz") fail("expected 404 ResponseException: ${res.asMap()}") } catch (e: ResponseException) { diff --git a/src/test/kotlin/org/opensearch/indexmanagement/transform/resthandler/RestExplainTransformActionIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/transform/resthandler/RestExplainTransformActionIT.kt index 2a66e30a5..b9af789d7 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/transform/resthandler/RestExplainTransformActionIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/transform/resthandler/RestExplainTransformActionIT.kt @@ -6,9 +6,10 @@ package org.opensearch.indexmanagement.transform.resthandler import org.junit.Assert +import org.opensearch.client.Request import org.opensearch.client.ResponseException import org.opensearch.core.rest.RestStatus -import org.opensearch.indexmanagement.IndexManagementPlugin +import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.TRANSFORM_BASE_URI import org.opensearch.indexmanagement.makeRequest import org.opensearch.indexmanagement.transform.TransformRestTestCase @@ -160,7 +161,8 @@ class RestExplainTransformActionIT : TransformRestTestCase() { @Throws(Exception::class) fun `test explain transform when config doesnt exist`() { - deleteIndex(IndexManagementPlugin.INDEX_MANAGEMENT_INDEX) + val deleteISMIndexRequest = Request("DELETE", "/$INDEX_MANAGEMENT_INDEX") + adminClient().performRequest(deleteISMIndexRequest) val responseExplicit = client().makeRequest("GET", "$TRANSFORM_BASE_URI/no_config_some_transform/_explain") val expectedResponse = mapOf("no_config_some_transform" to "Failed to search transform metadata") assertEquals("Non-existent transform didn't return null", expectedResponse, responseExplicit.asMap()) From 58add11901a3dc32dc8c31edbfce82befc2eab73 Mon Sep 17 00:00:00 2001 From: Hailong Cui Date: Tue, 17 Sep 2024 00:43:03 +0800 Subject: [PATCH 18/37] upgrade upload-artifact to version 3 (#1252) * upgrade upload-artifact to version 4 Signed-off-by: Hailong Cui * add overwrite as true Signed-off-by: Hailong Cui * use v3 as v4 need node.js 20 Signed-off-by: Hailong Cui --------- Signed-off-by: Hailong Cui --- .github/workflows/bwc-test-workflow.yml | 3 ++- .../workflows/docker-security-test-workflow.yml | 6 ++++-- .github/workflows/multi-node-test-workflow.yml | 3 ++- .github/workflows/security-test-workflow.yml | 3 ++- .github/workflows/test-and-build-workflow.yml | 15 +++++++++------ 5 files changed, 19 insertions(+), 11 deletions(-) diff --git a/.github/workflows/bwc-test-workflow.yml b/.github/workflows/bwc-test-workflow.yml index 656aea4cc..dc905d65d 100644 --- a/.github/workflows/bwc-test-workflow.yml +++ b/.github/workflows/bwc-test-workflow.yml @@ -26,8 +26,9 @@ jobs: echo "Running backwards compatibility tests..." ./gradlew bwcTestSuite - name: Upload failed logs - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 if: failure() with: name: logs + overwrite: 'true' path: build/testclusters/indexmanagementBwcCluster*/logs/* diff --git a/.github/workflows/docker-security-test-workflow.yml b/.github/workflows/docker-security-test-workflow.yml index 6511ebfec..de3d83ea9 100644 --- a/.github/workflows/docker-security-test-workflow.yml +++ b/.github/workflows/docker-security-test-workflow.yml @@ -84,10 +84,11 @@ jobs: echo "Security plugin is NOT available skipping this run as tests without security have already been run" fi - name: Upload failed logs - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 if: failure() with: name: logs + overwrite: 'true' path: build/testclusters/integTest-*/logs/* - name: Collect docker logs on failure uses: jwalton/gh-docker-logs@v2 @@ -96,8 +97,9 @@ jobs: - name: Tar logs run: tar cvzf ./logs.tgz ./logs - name: Upload logs to GitHub - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 if: failure() with: name: logs.tgz + overwrite: 'true' path: ./logs.tgz diff --git a/.github/workflows/multi-node-test-workflow.yml b/.github/workflows/multi-node-test-workflow.yml index f23add7cd..e3c1bb911 100644 --- a/.github/workflows/multi-node-test-workflow.yml +++ b/.github/workflows/multi-node-test-workflow.yml @@ -53,8 +53,9 @@ jobs: chown -R 1000:1000 `pwd` su `id -un 1000` -c "./gradlew integTest -PnumNodes=3 ${{ env.TEST_FILTER }}" - name: Upload failed logs - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 if: failure() with: name: logs + overwrite: 'true' path: build/testclusters/integTest-*/logs/* diff --git a/.github/workflows/security-test-workflow.yml b/.github/workflows/security-test-workflow.yml index d88750bda..8dc6eb19c 100644 --- a/.github/workflows/security-test-workflow.yml +++ b/.github/workflows/security-test-workflow.yml @@ -42,8 +42,9 @@ jobs: chown -R 1000:1000 `pwd` su `id -un 1000` -c "./gradlew integTest -Dsecurity=true -Dhttps=true --tests '*IT'" - name: Upload failed logs - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 if: failure() with: name: logs + overwrite: 'true' path: build/testclusters/integTest-*/logs/* diff --git a/.github/workflows/test-and-build-workflow.yml b/.github/workflows/test-and-build-workflow.yml index 73c24da42..a5dc4c0cb 100644 --- a/.github/workflows/test-and-build-workflow.yml +++ b/.github/workflows/test-and-build-workflow.yml @@ -56,10 +56,10 @@ jobs: chown -R 1000:1000 `pwd` su `id -un 1000` -c "./gradlew build ${{ env.TEST_FILTER }}" - name: Upload failed logs - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 if: ${{ failure() }} with: - name: logs + name: logs-${{ matrix.java }}-${{ matrix.feature }} path: build/testclusters/integTest-*/logs/* - name: Create Artifact Path run: | @@ -71,10 +71,12 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} # This step uses the upload-artifact Github action: https://github.com/actions/upload-artifact - name: Upload Artifacts - uses: actions/upload-artifact@v1 + # v4 requires node.js 20 which is not supported + uses: actions/upload-artifact@v3 with: - name: index-management-plugin-ubuntu-latest + name: index-management-plugin-ubuntu-latest-${{ matrix.java }} path: index-management-artifacts + overwrite: 'true' test-and-build-windows-macos: env: @@ -127,7 +129,8 @@ jobs: cp ./build/distributions/*.zip index-management-artifacts # This step uses the upload-artifact Github action: https://github.com/actions/upload-artifact - name: Upload Artifacts - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v4 with: - name: index-management-plugin-${{ matrix.os }} + name: index-management-plugin-${{ matrix.os }}-${{ matrix.java }} path: index-management-artifacts + overwrite: 'true' From 55633ee34016d8b4dbb7b5caa173e1ff3447d72d Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Mon, 16 Sep 2024 18:13:14 -0400 Subject: [PATCH 19/37] Remove 2 instances wildcard imports (#1251) --- .../resthandler/RestChangePolicyActionIT.kt | 2 +- .../rollup/resthandler/RestStartRollupActionIT.kt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestChangePolicyActionIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestChangePolicyActionIT.kt index 73e04f928..ee2621bca 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestChangePolicyActionIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestChangePolicyActionIT.kt @@ -36,7 +36,7 @@ import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StateMetaDa import org.opensearch.indexmanagement.waitFor import org.opensearch.rest.RestRequest import java.time.Instant -import java.util.* +import java.util.Locale class RestChangePolicyActionIT : IndexStateManagementRestTestCase() { private val testIndexName = javaClass.simpleName.lowercase(Locale.ROOT) diff --git a/src/test/kotlin/org/opensearch/indexmanagement/rollup/resthandler/RestStartRollupActionIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/rollup/resthandler/RestStartRollupActionIT.kt index 6baf8b474..e165a5b71 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/rollup/resthandler/RestStartRollupActionIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/rollup/resthandler/RestStartRollupActionIT.kt @@ -24,7 +24,7 @@ import org.opensearch.indexmanagement.waitFor import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule import java.time.Instant import java.time.temporal.ChronoUnit -import java.util.* +import java.util.Locale class RestStartRollupActionIT : RollupRestAPITestCase() { private val testName = javaClass.simpleName.lowercase(Locale.ROOT) From 809f85ca2fc26c7b1ee1fd3060bd35bf56b46520 Mon Sep 17 00:00:00 2001 From: Kshitij Tandon Date: Tue, 17 Sep 2024 17:41:00 +0530 Subject: [PATCH 20/37] Fixing snapshot bug (#1257) Signed-off-by: Kshitij Tandon Co-authored-by: bowenlan-amzn --- .../step/snapshot/WaitForSnapshotStep.kt | 47 +++++++-------- .../step/WaitForSnapshotStepTests.kt | 58 +++++++++---------- .../snapshotmanagement/TestUtils.kt | 25 ++++++-- 3 files changed, 72 insertions(+), 58 deletions(-) diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/snapshot/WaitForSnapshotStep.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/snapshot/WaitForSnapshotStep.kt index 36157f5c9..ef745d412 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/snapshot/WaitForSnapshotStep.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/snapshot/WaitForSnapshotStep.kt @@ -7,16 +7,16 @@ package org.opensearch.indexmanagement.indexstatemanagement.step.snapshot import org.apache.logging.log4j.LogManager import org.opensearch.ExceptionsHelper -import org.opensearch.action.admin.cluster.snapshots.status.SnapshotStatus -import org.opensearch.action.admin.cluster.snapshots.status.SnapshotsStatusRequest -import org.opensearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse -import org.opensearch.cluster.SnapshotsInProgress.State +import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest +import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse import org.opensearch.indexmanagement.indexstatemanagement.action.SnapshotAction import org.opensearch.indexmanagement.opensearchapi.suspendUntil import org.opensearch.indexmanagement.spi.indexstatemanagement.Step import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionProperties import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData +import org.opensearch.snapshots.SnapshotInfo +import org.opensearch.snapshots.SnapshotState import org.opensearch.transport.RemoteTransportException class WaitForSnapshotStep(private val action: SnapshotAction) : Step(name) { @@ -33,32 +33,31 @@ class WaitForSnapshotStep(private val action: SnapshotAction) : Step(name) { try { val snapshotName = getSnapshotName(managedIndexMetadata, indexName) ?: return this - val request = - SnapshotsStatusRequest() - .snapshots(arrayOf(snapshotName)) - .repository(repository) - val response: SnapshotsStatusResponse = context.client.admin().cluster().suspendUntil { snapshotsStatus(request, it) } - val status: SnapshotStatus? = + val newRequest = GetSnapshotsRequest() + .snapshots(arrayOf(snapshotName)) + .repository(repository) + val response: GetSnapshotsResponse = context.client.admin().cluster().suspendUntil { getSnapshots(newRequest, it) } + val status: SnapshotInfo? = response .snapshots - .find { snapshotStatus -> - snapshotStatus.snapshot.snapshotId.name == snapshotName && snapshotStatus.snapshot.repository == repository + .find { snapshotInfo -> + snapshotInfo.snapshotId().name == snapshotName } if (status != null) { - when (status.state) { - State.INIT, State.STARTED -> { + when (status.state()) { + SnapshotState.IN_PROGRESS -> { stepStatus = StepStatus.CONDITION_NOT_MET - info = mapOf("message" to getSnapshotInProgressMessage(indexName), "state" to status.state.name) + info = mapOf("message" to getSnapshotInProgressMessage(indexName), "state" to status.state().toString()) } - State.SUCCESS -> { + SnapshotState.SUCCESS -> { stepStatus = StepStatus.COMPLETED - info = mapOf("message" to getSuccessMessage(indexName), "state" to status.state.name) + info = mapOf("message" to getSuccessMessage(indexName), "state" to status.state().toString()) } else -> { // State.FAILED, State.ABORTED val message = getFailedExistsMessage(indexName) logger.warn(message) stepStatus = StepStatus.FAILED - info = mapOf("message" to message, "state" to status.state.name) + info = mapOf("message" to message, "state" to status.state().toString()) } } } else { @@ -98,13 +97,11 @@ class WaitForSnapshotStep(private val action: SnapshotAction) : Step(name) { return actionProperties.snapshotName } - override fun getUpdatedManagedIndexMetadata(currentMetadata: ManagedIndexMetaData): ManagedIndexMetaData { - return currentMetadata.copy( - stepMetaData = StepMetaData(name, getStepStartTime(currentMetadata).toEpochMilli(), stepStatus), - transitionTo = null, - info = info, - ) - } + override fun getUpdatedManagedIndexMetadata(currentMetadata: ManagedIndexMetaData): ManagedIndexMetaData = currentMetadata.copy( + stepMetaData = StepMetaData(name, getStepStartTime(currentMetadata).toEpochMilli(), stepStatus), + transitionTo = null, + info = info, + ) override fun isIdempotent(): Boolean = true diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/WaitForSnapshotStepTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/WaitForSnapshotStepTests.kt index ae4d394df..faed2f2f1 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/WaitForSnapshotStepTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/WaitForSnapshotStepTests.kt @@ -11,17 +11,17 @@ import com.nhaarman.mockitokotlin2.doReturn import com.nhaarman.mockitokotlin2.mock import com.nhaarman.mockitokotlin2.whenever import kotlinx.coroutines.runBlocking -import org.opensearch.action.admin.cluster.snapshots.status.SnapshotStatus -import org.opensearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse +import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse import org.opensearch.client.AdminClient import org.opensearch.client.Client import org.opensearch.client.ClusterAdminClient -import org.opensearch.cluster.SnapshotsInProgress import org.opensearch.cluster.service.ClusterService import org.opensearch.common.settings.Settings import org.opensearch.core.action.ActionListener import org.opensearch.indexmanagement.indexstatemanagement.action.SnapshotAction import org.opensearch.indexmanagement.indexstatemanagement.step.snapshot.WaitForSnapshotStep +import org.opensearch.indexmanagement.snapshotmanagement.mockInProgressSnapshotInfo +import org.opensearch.indexmanagement.snapshotmanagement.mockSnapshotInfo import org.opensearch.indexmanagement.spi.indexstatemanagement.Step import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionMetaData import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionProperties @@ -29,8 +29,8 @@ import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedInde import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext import org.opensearch.jobscheduler.spi.utils.LockService import org.opensearch.script.ScriptService -import org.opensearch.snapshots.Snapshot -import org.opensearch.snapshots.SnapshotId +import org.opensearch.snapshots.SnapshotInfo +import org.opensearch.snapshots.SnapshotState import org.opensearch.test.OpenSearchTestCase import org.opensearch.transport.RemoteTransportException @@ -70,13 +70,10 @@ class WaitForSnapshotStepTests : OpenSearchTestCase() { } fun `test snapshot status states`() { - val snapshotStatus: SnapshotStatus = mock() - val response: SnapshotsStatusResponse = mock() - whenever(response.snapshots).doReturn(listOf(snapshotStatus)) - whenever(snapshotStatus.snapshot).doReturn(Snapshot("repo", SnapshotId("snapshot-name", "some_uuid"))) + val snapshotInfo: SnapshotInfo = mockInProgressSnapshotInfo(snapshot) + val response: GetSnapshotsResponse = mock() + whenever(response.snapshots).doReturn(listOf(snapshotInfo)) val client = getClient(getAdminClient(getClusterAdminClient(response, null))) - - whenever(snapshotStatus.state).doReturn(SnapshotsInProgress.State.INIT) runBlocking { val snapshotAction = SnapshotAction("repo", snapshot, 0) val metadata = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, ActionMetaData(WaitForSnapshotStep.name, 1, 0, false, 0, null, ActionProperties(snapshotName = "snapshot-name")), null, null, null) @@ -88,7 +85,8 @@ class WaitForSnapshotStepTests : OpenSearchTestCase() { assertEquals("Did not get snapshot in progress message", WaitForSnapshotStep.getSnapshotInProgressMessage("test"), updatedManagedIndexMetaData.info!!["message"]) } - whenever(snapshotStatus.state).doReturn(SnapshotsInProgress.State.STARTED) + val snapshotInfo2: SnapshotInfo = mockSnapshotInfo(snapshot, SnapshotState.SUCCESS) + whenever(response.snapshots).doReturn(listOf(snapshotInfo2)) runBlocking { val snapshotAction = SnapshotAction("repo", snapshot, 0) val metadata = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, ActionMetaData(WaitForSnapshotStep.name, 1, 0, false, 0, null, ActionProperties(snapshotName = "snapshot-name")), null, null, null) @@ -96,11 +94,12 @@ class WaitForSnapshotStepTests : OpenSearchTestCase() { val context = StepContext(metadata, clusterService, client, null, null, scriptService, settings, lockService) step.preExecute(logger, context).execute() val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetadata(metadata) - assertEquals("Step status is not CONDITION_NOT_MET", Step.StepStatus.CONDITION_NOT_MET, updatedManagedIndexMetaData.stepMetaData?.stepStatus) - assertEquals("Did not get snapshot in progress message", WaitForSnapshotStep.getSnapshotInProgressMessage("test"), updatedManagedIndexMetaData.info!!["message"]) + assertEquals("Step status is not COMPLETED", Step.StepStatus.COMPLETED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) + assertEquals("Did not get snapshot completed message", WaitForSnapshotStep.getSuccessMessage("test"), updatedManagedIndexMetaData.info!!["message"]) } - whenever(snapshotStatus.state).doReturn(SnapshotsInProgress.State.SUCCESS) + val snapshotInfo3: SnapshotInfo = mockSnapshotInfo(snapshot, SnapshotState.FAILED) + whenever(response.snapshots).doReturn(listOf(snapshotInfo3)) runBlocking { val snapshotAction = SnapshotAction("repo", snapshot, 0) val metadata = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, ActionMetaData(WaitForSnapshotStep.name, 1, 0, false, 0, null, ActionProperties(snapshotName = "snapshot-name")), null, null, null) @@ -108,11 +107,12 @@ class WaitForSnapshotStepTests : OpenSearchTestCase() { val context = StepContext(metadata, clusterService, client, null, null, scriptService, settings, lockService) step.preExecute(logger, context).execute() val updatedManagedIndexMetaData = step.getUpdatedManagedIndexMetadata(metadata) - assertEquals("Step status is not COMPLETED", Step.StepStatus.COMPLETED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) - assertEquals("Did not get snapshot completed message", WaitForSnapshotStep.getSuccessMessage("test"), updatedManagedIndexMetaData.info!!["message"]) + assertEquals("Step status is not FAILED", Step.StepStatus.FAILED, updatedManagedIndexMetaData.stepMetaData?.stepStatus) + assertEquals("Did not get snapshot failed message", WaitForSnapshotStep.getFailedExistsMessage("test"), updatedManagedIndexMetaData.info!!["message"]) } - whenever(snapshotStatus.state).doReturn(SnapshotsInProgress.State.ABORTED) + val snapshotInfo4: SnapshotInfo = mockSnapshotInfo(snapshot, SnapshotState.PARTIAL) + whenever(response.snapshots).doReturn(listOf(snapshotInfo4)) runBlocking { val snapshotAction = SnapshotAction("repo", snapshot, 0) val metadata = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, ActionMetaData(WaitForSnapshotStep.name, 1, 0, false, 0, null, ActionProperties(snapshotName = "snapshot-name")), null, null, null) @@ -124,7 +124,8 @@ class WaitForSnapshotStepTests : OpenSearchTestCase() { assertEquals("Did not get snapshot failed message", WaitForSnapshotStep.getFailedExistsMessage("test"), updatedManagedIndexMetaData.info!!["message"]) } - whenever(snapshotStatus.state).doReturn(SnapshotsInProgress.State.FAILED) + val snapshotInfo5: SnapshotInfo = mockSnapshotInfo(snapshot, SnapshotState.INCOMPATIBLE) + whenever(response.snapshots).doReturn(listOf(snapshotInfo5)) runBlocking { val snapshotAction = SnapshotAction("repo", snapshot, 0) val metadata = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, ActionMetaData(WaitForSnapshotStep.name, 1, 0, false, 0, null, ActionProperties(snapshotName = "snapshot-name")), null, null, null) @@ -138,10 +139,9 @@ class WaitForSnapshotStepTests : OpenSearchTestCase() { } fun `test snapshot not in response list`() { - val snapshotStatus: SnapshotStatus = mock() - val response: SnapshotsStatusResponse = mock() - whenever(response.snapshots).doReturn(listOf(snapshotStatus)) - whenever(snapshotStatus.snapshot).doReturn(Snapshot("repo", SnapshotId("snapshot-different-name", "some_uuid"))) + val snapshotInfo: SnapshotInfo = mockSnapshotInfo("snapshot-different-name") + val response: GetSnapshotsResponse = mock() + whenever(response.snapshots).doReturn(listOf(snapshotInfo)) val client = getClient(getAdminClient(getClusterAdminClient(response, null))) runBlocking { @@ -190,17 +190,17 @@ class WaitForSnapshotStepTests : OpenSearchTestCase() { private fun getAdminClient(clusterAdminClient: ClusterAdminClient): AdminClient = mock { on { cluster() } doReturn clusterAdminClient } - private fun getClusterAdminClient(snapshotsStatusResponse: SnapshotsStatusResponse?, exception: Exception?): ClusterAdminClient { - assertTrue("Must provide one and only one response or exception", (snapshotsStatusResponse != null).xor(exception != null)) + private fun getClusterAdminClient(getSnapshotsResponse: GetSnapshotsResponse?, exception: Exception?): ClusterAdminClient { + assertTrue("Must provide one and only one response or exception", (getSnapshotsResponse != null).xor(exception != null)) return mock { doAnswer { invocationOnMock -> - val listener = invocationOnMock.getArgument>(1) - if (snapshotsStatusResponse != null) { - listener.onResponse(snapshotsStatusResponse) + val listener = invocationOnMock.getArgument>(1) + if (getSnapshotsResponse != null) { + listener.onResponse(getSnapshotsResponse) } else { listener.onFailure(exception) } - }.whenever(this.mock).snapshotsStatus(any(), any()) + }.whenever(this.mock).getSnapshots(any(), any()) } } } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/TestUtils.kt b/src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/TestUtils.kt index 82344ac8c..a610039ad 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/TestUtils.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/TestUtils.kt @@ -37,6 +37,7 @@ import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule import org.opensearch.snapshots.Snapshot import org.opensearch.snapshots.SnapshotId import org.opensearch.snapshots.SnapshotInfo +import org.opensearch.snapshots.SnapshotState import org.opensearch.test.OpenSearchTestCase.randomAlphaOfLength import org.opensearch.test.OpenSearchTestCase.randomBoolean import org.opensearch.test.OpenSearchTestCase.randomIntBetween @@ -177,11 +178,14 @@ fun randomSMState(): SMState = SMState.values()[randomIntBetween(0, SMState.valu fun randomNotificationConfig(): NotificationConfig = NotificationConfig(randomChannel(), randomConditions()) -fun randomConditions(): NotificationConfig.Conditions = NotificationConfig.Conditions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()) +fun randomConditions(): NotificationConfig.Conditions = + NotificationConfig.Conditions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()) -fun ToXContent.toJsonString(params: ToXContent.Params = ToXContent.EMPTY_PARAMS): String = this.toXContent(XContentFactory.jsonBuilder(), params).string() +fun ToXContent.toJsonString(params: ToXContent.Params = ToXContent.EMPTY_PARAMS): String = + this.toXContent(XContentFactory.jsonBuilder(), params).string() -fun ToXContent.toMap(params: ToXContent.Params = ToXContent.EMPTY_PARAMS): Map = this.toXContent(XContentFactory.jsonBuilder(), params).toMap() +fun ToXContent.toMap(params: ToXContent.Params = ToXContent.EMPTY_PARAMS): Map = + this.toXContent(XContentFactory.jsonBuilder(), params).toMap() fun mockIndexResponse(status: RestStatus = RestStatus.OK): IndexResponse { val indexResponse: IndexResponse = mock() @@ -263,6 +267,18 @@ fun mockInProgressSnapshotInfo( return SnapshotInfo(entry) } +fun mockSnapshotInfo( + name: String = randomAlphaOfLength(10), + snapshotState: SnapshotState, +): SnapshotInfo { + return SnapshotInfo( + SnapshotId(name, UUIDs.randomBase64UUID()), + emptyList(), + emptyList(), + snapshotState, + ) +} + fun mockGetSnapshotResponse(num: Int): GetSnapshotsResponse { val getSnapshotsRes: GetSnapshotsResponse = mock() whenever(getSnapshotsRes.snapshots).doReturn(mockSnapshotInfoList(num)) @@ -281,4 +297,5 @@ fun mockSnapshotInfoList(num: Int, namePrefix: String = randomAlphaOfLength(10)) return result.toList() } -fun String.parser(): XContentParser = XContentType.JSON.xContent().createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, this) +fun String.parser(): XContentParser = + XContentType.JSON.xContent().createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, this) From 316df7006b23c5f809eb6594bffd46587cb6e156 Mon Sep 17 00:00:00 2001 From: Hailong Cui Date: Wed, 18 Sep 2024 00:44:37 +0800 Subject: [PATCH 21/37] bump bwc version after 2.17 release (#1259) --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 6ed9b7091..d3d344700 100644 --- a/build.gradle +++ b/build.gradle @@ -635,7 +635,7 @@ task integTestRemote(type: RestIntegTestTask) { // === Set up BWC tests === -String bwcVersionShort = "2.12.0" +String bwcVersionShort = "2.18.0" String bwcVersion = bwcVersionShort + ".0" String baseName = "indexmanagementBwcCluster" From 4adc99edbcc68bf8197ffad82d611a76451481c4 Mon Sep 17 00:00:00 2001 From: Kshitij Tandon Date: Mon, 23 Sep 2024 16:20:37 +0530 Subject: [PATCH 22/37] Move non-active maintainer to emeritus (#1263) Signed-off-by: Kshitij Tandon --- .github/CODEOWNERS | 2 +- MAINTAINERS.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 67a22979a..cd9176ea7 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -* @bowenlan-amzn @Hailong-am @r1walz @vikasvb90 +* @bowenlan-amzn @Hailong-am @vikasvb90 diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 60cc5d0f4..8127dcfa5 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -9,7 +9,6 @@ This document contains a list of maintainers in this repo. See [opensearch-proje | Vikas Bansal | [vikasvb90](https://github.com/vikasvb90) | Amazon | | Bowen Lan | [bowenlan-amzn](https://github.com/bowenlan-amzn) | Amazon | | Hailong Cui | [Hailong-am](https://github.com/Hailong-am) | Amazon | -| Rohit Ashiwal | [r1walz](https://github.com/r1walz) | Independent | ## Emeritus @@ -25,3 +24,4 @@ This document contains a list of maintainers in this repo. See [opensearch-proje | Surya Sashank Nistala | [eirsep](https://github.com/eirsep) | Amazon | | Thomas Hurney | [AWSHurneyt](https://github.com/AWSHurneyt) | Amazon | | Xuesong Luo | [xluo-aws](https://github.com/xluo-aws) | Amazon | +| Rohit Ashiwal | [r1walz](https://github.com/r1walz) | Independent | From 335bd4c71ff489d52de0ee99a972b761cb650c60 Mon Sep 17 00:00:00 2001 From: Kshitij Tandon Date: Tue, 29 Oct 2024 11:30:52 +0530 Subject: [PATCH 23/37] Updating baseline JDK version to JDK-21 (#1276) * Updating baseline JDK version to JDK-21 Signed-off-by: Kshitij Tandon * Updating kotlin version Signed-off-by: Kshitij Tandon * Updating some versions in build.gradle to support JDK 21 Signed-off-by: Kshitij Tandon * Updating some more versions and resolving some dependency conflicts Signed-off-by: Kshitij Tandon --------- Signed-off-by: Kshitij Tandon --- .github/workflows/maven-publish.yml | 2 +- .../workflows/multi-node-test-workflow.yml | 2 +- .github/workflows/test-and-build-workflow.yml | 4 ++-- DEVELOPER_GUIDE.md | 10 +++++----- build.gradle | 19 +++++++++++-------- detekt.yml | 4 ++++ .../ManagedIndexRunner.kt | 3 +++ .../forcemerge/AttemptCallForceMergeStep.kt | 2 ++ 8 files changed, 29 insertions(+), 17 deletions(-) diff --git a/.github/workflows/maven-publish.yml b/.github/workflows/maven-publish.yml index 5e947b345..0cb12857a 100644 --- a/.github/workflows/maven-publish.yml +++ b/.github/workflows/maven-publish.yml @@ -20,7 +20,7 @@ jobs: - uses: actions/setup-java@v3 with: distribution: temurin # Temurin is a distribution of adoptium - java-version: 11 + java-version: 21 - uses: actions/checkout@v3 - uses: aws-actions/configure-aws-credentials@v1 with: diff --git a/.github/workflows/multi-node-test-workflow.yml b/.github/workflows/multi-node-test-workflow.yml index e3c1bb911..40d9cc905 100644 --- a/.github/workflows/multi-node-test-workflow.yml +++ b/.github/workflows/multi-node-test-workflow.yml @@ -44,7 +44,7 @@ jobs: uses: actions/setup-java@v2 with: distribution: temurin # Temurin is a distribution of adoptium - java-version: 17 + java-version: 21 # index-management - name: Checkout Branch uses: actions/checkout@v2 diff --git a/.github/workflows/test-and-build-workflow.yml b/.github/workflows/test-and-build-workflow.yml index a5dc4c0cb..5713aea5e 100644 --- a/.github/workflows/test-and-build-workflow.yml +++ b/.github/workflows/test-and-build-workflow.yml @@ -24,7 +24,7 @@ jobs: fail-fast: false # This starts three jobs, setting these environment variables uniquely for the different jobs matrix: - java: [11, 17, 21] + java: [21] feature: [ism, non-ism] include: - feature: ism @@ -88,7 +88,7 @@ jobs: fail-fast: false # This starts three jobs, setting these environment variables uniquely for the different jobs matrix: - java: [11, 17, 21] + java: [21] os: [windows-latest, macos-latest] feature: [ism, non-ism] include: diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index c37625806..690727245 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -1,7 +1,7 @@ - [Developer Guide](#developer-guide) - [Forking and Cloning](#forking-and-cloning) - [Install Prerequisites](#install-prerequisites) - - [JDK 11](#jdk-11) + - [JDK 21](#jdk-21) - [Setup](#setup) - [Build](#build) - [Building from the command line](#building-from-the-command-line) @@ -19,17 +19,17 @@ Fork this repository on GitHub, and clone locally with `git clone`. ### Install Prerequisites -#### JDK 11 +#### JDK 21 -OpenSearch components build using Java 11 at a minimum. This means you must have a JDK 11 installed with the environment variable `JAVA_HOME` referencing the path to Java home for your JDK 11 installation, e.g. `JAVA_HOME=/usr/lib/jvm/jdk-11`. +OpenSearch components build using Java 21 at a minimum. This means you must have a JDK 21 installed with the environment variable `JAVA_HOME` referencing the path to Java home for your JDK 21 installation, e.g. `JAVA_HOME=/usr/lib/jvm/jdk-21`. -Download Java 11 from [here](https://adoptium.net/releases.html?variant=openjdk11). +Download Java 21 from [here](https://adoptium.net/releases.html?variant=openjdk21). ## Setup 1. Check out this package from version control. 2. Launch Intellij IDEA, choose **Import Project**, and select the `settings.gradle` file in the root of this package. -3. To build from the command line, set `JAVA_HOME` to point to a JDK >= 11 before running `./gradlew`. +3. To build from the command line, set `JAVA_HOME` to point to a JDK >= 21 before running `./gradlew`. - Unix System 1. `export JAVA_HOME=jdk-install-dir`: Replace `jdk-install-dir` with the JAVA_HOME directory of your system. 2. `export PATH=$JAVA_HOME/bin:$PATH` diff --git a/build.gradle b/build.gradle index d3d344700..5ab7defc4 100644 --- a/build.gradle +++ b/build.gradle @@ -63,7 +63,7 @@ buildscript { notifications_core_build_download = 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/' + opensearch_no_snapshot + '/latest/linux/x64/tar/builds/opensearch/plugins/opensearch-notifications-core-' + notifications_no_snapshot + '.zip' - kotlin_version = System.getProperty("kotlin.version", "1.8.21") + kotlin_version = System.getProperty("kotlin.version", "1.9.25") security_plugin_version = System.getProperty("security.version", opensearch_build) } @@ -79,8 +79,8 @@ buildscript { classpath "org.opensearch.gradle:build-tools:${opensearch_version}" classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:${kotlin_version}" classpath "org.jetbrains.kotlin:kotlin-allopen:${kotlin_version}" - classpath "io.gitlab.arturbosch.detekt:detekt-gradle-plugin:1.21.0" - classpath "org.jacoco:org.jacoco.agent:0.8.7" + classpath "io.gitlab.arturbosch.detekt:detekt-gradle-plugin:1.23.6" + classpath "org.jacoco:org.jacoco.agent:0.8.10" } } @@ -105,6 +105,9 @@ configurations.all { resolutionStrategy { force "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}" force "org.jetbrains.kotlin:kotlin-stdlib-common:${kotlin_version}" + force "org.jetbrains.kotlin:kotlin-reflect:${kotlin_version}" + force "org.jetbrains.kotlin:kotlin-stdlib-jdk8:${kotlin_version}" + force "org.jetbrains.kotlin:kotlin-stdlib-jdk7:${kotlin_version}" force 'junit:junit:4.13.1' force 'commons-beanutils:commons-beanutils:1.9.4' force 'com.google.guava:guava:30.0-jre' @@ -202,7 +205,7 @@ dependencies { implementation "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}" implementation "org.jetbrains.kotlin:kotlin-stdlib-common:${kotlin_version}" implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:${kotlin_version}" - implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.3.9' + implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.3' implementation "org.jetbrains:annotations:13.0" implementation project(path: ":${rootProject.name}-spi", configuration: 'shadow') implementation "org.opensearch:common-utils:${common_utils_version}" @@ -222,7 +225,7 @@ dependencies { } } configurations.ktlint { - resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.8.22" + resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10" } // https://aws.oss.sonatype.org/content/repositories/snapshots/org/opensearch/plugin/ @@ -274,11 +277,11 @@ publishing { tasks.generatePomFileForPluginZipPublication.dependsOn publishNebulaPublicationToMavenLocal plugins.withId('java') { - sourceCompatibility = targetCompatibility = JavaVersion.VERSION_11 + sourceCompatibility = targetCompatibility = JavaVersion.VERSION_21 } plugins.withId('org.jetbrains.kotlin.jvm') { - compileKotlin.kotlinOptions.jvmTarget = compileTestKotlin.kotlinOptions.jvmTarget = JavaVersion.VERSION_11 + compileKotlin.kotlinOptions.jvmTarget = compileTestKotlin.kotlinOptions.jvmTarget = JavaVersion.VERSION_21 compileKotlin.dependsOn ktlint } @@ -892,4 +895,4 @@ task updateVersion { // String tokenization to support -SNAPSHOT ant.replaceregexp(file: 'build.gradle', match: '"opensearch.version", "\\d.*"', replace: '"opensearch.version", "' + newVersion.tokenize('-')[0] + '-SNAPSHOT"', flags: 'g', byline: true) } -} +} \ No newline at end of file diff --git a/detekt.yml b/detekt.yml index 57ab2f3c5..620bd4998 100644 --- a/detekt.yml +++ b/detekt.yml @@ -15,6 +15,10 @@ style: excludes: ['**/test/**'] FunctionOnlyReturningConstant: active: false + UseRequire: + active: false + UnusedPrivateProperty: + active: false complexity: LargeClass: diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/ManagedIndexRunner.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/ManagedIndexRunner.kt index 003a612ce..890e5008d 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/ManagedIndexRunner.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/ManagedIndexRunner.kt @@ -7,6 +7,7 @@ package org.opensearch.indexmanagement.indexstatemanagement import kotlinx.coroutines.CoroutineName import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.DelicateCoroutinesApi import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.GlobalScope import kotlinx.coroutines.SupervisorJob @@ -254,6 +255,7 @@ object ManagedIndexRunner : } } + @OptIn(DelicateCoroutinesApi::class) @Suppress("ReturnCount", "ComplexMethod", "LongMethod", "ComplexCondition", "NestedBlockDepth") private suspend fun runManagedIndexConfig(managedIndexConfig: ManagedIndexConfig, jobContext: JobExecutionContext) { logger.debug("Run job for index ${managedIndexConfig.index}") @@ -640,6 +642,7 @@ object ManagedIndexRunner : * update metadata in config index, and save metadata in history after update * this can be called 2 times in one job run, so need to save seqNo & primeTerm */ + @OptIn(DelicateCoroutinesApi::class) private suspend fun updateManagedIndexMetaData( managedIndexMetaData: ManagedIndexMetaData, lastUpdateResult: UpdateMetadataResult? = null, diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/forcemerge/AttemptCallForceMergeStep.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/forcemerge/AttemptCallForceMergeStep.kt index 3ff796e55..e7596842a 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/forcemerge/AttemptCallForceMergeStep.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/forcemerge/AttemptCallForceMergeStep.kt @@ -6,6 +6,7 @@ package org.opensearch.indexmanagement.indexstatemanagement.step.forcemerge import kotlinx.coroutines.CoroutineName +import kotlinx.coroutines.DelicateCoroutinesApi import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.GlobalScope import kotlinx.coroutines.delay @@ -30,6 +31,7 @@ class AttemptCallForceMergeStep(private val action: ForceMergeAction) : Step(nam private var stepStatus = StepStatus.STARTING private var info: Map? = null + @OptIn(DelicateCoroutinesApi::class) @Suppress("TooGenericExceptionCaught", "ComplexMethod") override suspend fun execute(): AttemptCallForceMergeStep { val context = this.context ?: return this From dd710e6367cfea34fe1f6a179bf2e1eb742ee8cb Mon Sep 17 00:00:00 2001 From: Kshitij Tandon Date: Tue, 29 Oct 2024 16:16:16 +0530 Subject: [PATCH 24/37] Allowing non-rollup and rollup indices to be searched together (#1268) * Allowing non-rollup and rollup indices to be searched together Signed-off-by: Kshitij Tandon * Fixing an issue in the integration test Signed-off-by: Kshitij Tandon * Using trace in place of warn in logger Signed-off-by: Kshitij Tandon --------- Signed-off-by: Kshitij Tandon --- .../indexmanagement/IndexManagementPlugin.kt | 1 + .../rollup/interceptor/RollupInterceptor.kt | 25 +++++++++++----- .../rollup/settings/RollupSettings.kt | 9 ++++++ .../IndexManagementSettingsTests.kt | 2 ++ .../rollup/RollupRestTestCase.kt | 18 ++++++++++++ .../rollup/interceptor/RollupInterceptorIT.kt | 29 +++++++++++++++++-- 6 files changed, 73 insertions(+), 11 deletions(-) diff --git a/src/main/kotlin/org/opensearch/indexmanagement/IndexManagementPlugin.kt b/src/main/kotlin/org/opensearch/indexmanagement/IndexManagementPlugin.kt index e59602205..981a59b67 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/IndexManagementPlugin.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/IndexManagementPlugin.kt @@ -534,6 +534,7 @@ class IndexManagementPlugin : JobSchedulerExtension, NetworkPlugin, ActionPlugin RollupSettings.ROLLUP_SEARCH_ENABLED, RollupSettings.ROLLUP_DASHBOARDS, RollupSettings.ROLLUP_SEARCH_ALL_JOBS, + RollupSettings.ROLLUP_SEARCH_SOURCE_INDICES, TransformSettings.TRANSFORM_JOB_INDEX_BACKOFF_COUNT, TransformSettings.TRANSFORM_JOB_INDEX_BACKOFF_MILLIS, TransformSettings.TRANSFORM_JOB_SEARCH_BACKOFF_COUNT, diff --git a/src/main/kotlin/org/opensearch/indexmanagement/rollup/interceptor/RollupInterceptor.kt b/src/main/kotlin/org/opensearch/indexmanagement/rollup/interceptor/RollupInterceptor.kt index 87919c173..694d2646c 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/rollup/interceptor/RollupInterceptor.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/rollup/interceptor/RollupInterceptor.kt @@ -63,6 +63,8 @@ class RollupInterceptor( @Volatile private var searchAllJobs = RollupSettings.ROLLUP_SEARCH_ALL_JOBS.get(settings) + @Volatile private var searchRawRollupIndices = RollupSettings.ROLLUP_SEARCH_SOURCE_INDICES.get(settings) + init { clusterService.clusterSettings.addSettingsUpdateConsumer(RollupSettings.ROLLUP_SEARCH_ENABLED) { searchEnabled = it @@ -70,6 +72,9 @@ class RollupInterceptor( clusterService.clusterSettings.addSettingsUpdateConsumer(RollupSettings.ROLLUP_SEARCH_ALL_JOBS) { searchAllJobs = it } + clusterService.clusterSettings.addSettingsUpdateConsumer(RollupSettings.ROLLUP_SEARCH_SOURCE_INDICES) { + searchRawRollupIndices = it + } } @Suppress("SpreadOperator") @@ -144,15 +149,16 @@ class RollupInterceptor( private fun validateIndicies(concreteIndices: Array, fieldMappings: Set): Map> { var allMatchingRollupJobs: Map> = mapOf() for (concreteIndex in concreteIndices) { - val rollupJobs = - clusterService.state().metadata.index(concreteIndex).getRollupJobs() - ?: throw IllegalArgumentException("Not all indices have rollup job") - - val (matchingRollupJobs, issues) = findMatchingRollupJobs(fieldMappings, rollupJobs) - if (issues.isNotEmpty() || matchingRollupJobs.isEmpty()) { - throw IllegalArgumentException("Could not find a rollup job that can answer this query because $issues") + val rollupJobs = clusterService.state().metadata.index(concreteIndex).getRollupJobs() + if (rollupJobs != null) { + val (matchingRollupJobs, issues) = findMatchingRollupJobs(fieldMappings, rollupJobs) + if (issues.isNotEmpty() || matchingRollupJobs.isEmpty()) { + throw IllegalArgumentException("Could not find a rollup job that can answer this query because $issues") + } + allMatchingRollupJobs += matchingRollupJobs + } else if (!searchRawRollupIndices) { + throw IllegalArgumentException("Not all indices have rollup job") } - allMatchingRollupJobs += matchingRollupJobs } return allMatchingRollupJobs } @@ -347,6 +353,9 @@ class RollupInterceptor( if (searchAllJobs) { request.source(request.source().rewriteSearchSourceBuilder(matchingRollupJobs.keys, fieldNameMappingTypeMap, concreteSourceIndex)) } else { + if (matchingRollupJobs.keys.size > 1) { + logger.trace("Trying search with search across multiple rollup jobs disabled so will give result with largest rollup window") + } request.source(request.source().rewriteSearchSourceBuilder(matchedRollup, fieldNameMappingTypeMap, concreteSourceIndex)) } } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/rollup/settings/RollupSettings.kt b/src/main/kotlin/org/opensearch/indexmanagement/rollup/settings/RollupSettings.kt index 0554a7061..476fcd32c 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/rollup/settings/RollupSettings.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/rollup/settings/RollupSettings.kt @@ -13,6 +13,7 @@ class RollupSettings { companion object { const val DEFAULT_ROLLUP_ENABLED = true const val DEFAULT_SEARCH_ALL_JOBS = false + const val DEFAULT_SEARCH_SOURCE_INDICES = false const val DEFAULT_ACQUIRE_LOCK_RETRY_COUNT = 3 const val DEFAULT_ACQUIRE_LOCK_RETRY_DELAY = 1000L const val DEFAULT_RENEW_LOCK_RETRY_COUNT = 3 @@ -85,6 +86,14 @@ class RollupSettings { Setting.Property.Dynamic, ) + val ROLLUP_SEARCH_SOURCE_INDICES: Setting = + Setting.boolSetting( + "plugins.rollup.search.search_source_indices", + DEFAULT_SEARCH_SOURCE_INDICES, + Setting.Property.NodeScope, + Setting.Property.Dynamic, + ) + val ROLLUP_DASHBOARDS: Setting = Setting.boolSetting( "plugins.rollup.dashboards.enabled", diff --git a/src/test/kotlin/org/opensearch/indexmanagement/IndexManagementSettingsTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/IndexManagementSettingsTests.kt index 55fce2adc..2546e6271 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/IndexManagementSettingsTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/IndexManagementSettingsTests.kt @@ -91,6 +91,7 @@ class IndexManagementSettingsTests : OpenSearchTestCase() { RollupSettings.ROLLUP_ENABLED, RollupSettings.ROLLUP_SEARCH_ENABLED, RollupSettings.ROLLUP_SEARCH_ALL_JOBS, + RollupSettings.ROLLUP_SEARCH_SOURCE_INDICES, RollupSettings.ROLLUP_DASHBOARDS, SnapshotManagementSettings.FILTER_BY_BACKEND_ROLES, ), @@ -176,6 +177,7 @@ class IndexManagementSettingsTests : OpenSearchTestCase() { assertEquals(RollupSettings.ROLLUP_ENABLED.get(settings), false) assertEquals(RollupSettings.ROLLUP_SEARCH_ENABLED.get(settings), false) assertEquals(RollupSettings.ROLLUP_SEARCH_ALL_JOBS.get(settings), false) + assertEquals(RollupSettings.ROLLUP_SEARCH_SOURCE_INDICES.get(settings), false) assertEquals(RollupSettings.ROLLUP_INGEST_BACKOFF_MILLIS.get(settings), TimeValue.timeValueMillis(1)) assertEquals(RollupSettings.ROLLUP_INGEST_BACKOFF_COUNT.get(settings), 1) assertEquals(RollupSettings.ROLLUP_SEARCH_BACKOFF_MILLIS.get(settings), TimeValue.timeValueMillis(1)) diff --git a/src/test/kotlin/org/opensearch/indexmanagement/rollup/RollupRestTestCase.kt b/src/test/kotlin/org/opensearch/indexmanagement/rollup/RollupRestTestCase.kt index a1fec9755..6e527abfb 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/rollup/RollupRestTestCase.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/rollup/RollupRestTestCase.kt @@ -269,6 +269,24 @@ abstract class RollupRestTestCase : IndexManagementRestTestCase() { assertEquals("Request failed", RestStatus.OK, res.restStatus()) } + protected fun updateSearchRawRollupClusterSetting(value: Boolean) { + val formattedValue = "\"${value}\"" + val request = + """ + { + "persistent": { + "${RollupSettings.ROLLUP_SEARCH_SOURCE_INDICES.key}": $formattedValue + } + } + """.trimIndent() + val res = + client().makeRequest( + "PUT", "_cluster/settings", emptyMap(), + StringEntity(request, ContentType.APPLICATION_JSON), + ) + assertEquals("Request failed", RestStatus.OK, res.restStatus()) + } + protected fun createSampleIndexForQSQTest(index: String) { val mapping = """ diff --git a/src/test/kotlin/org/opensearch/indexmanagement/rollup/interceptor/RollupInterceptorIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/rollup/interceptor/RollupInterceptorIT.kt index b87fe55ae..8f658b885 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/rollup/interceptor/RollupInterceptorIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/rollup/interceptor/RollupInterceptorIT.kt @@ -1092,12 +1092,12 @@ class RollupInterceptorIT : RollupRestTestCase() { }, "aggs": { "sum_passenger_count": { "sum": { "field": "passenger_count" } }, - "max_passenger_count": { "max": { "field": "passenger_count" } }, - "value_count_passenger_count": { "value_count": { "field": "passenger_count" } } + "max_passenger_count": { "max": { "field": "passenger_count" } } } } """.trimIndent() - // Search 1 non-rollup index and 1 rollup +// Search 1 non-rollup index and 1 rollup + updateSearchRawRollupClusterSetting(false) val searchResult1 = client().makeRequest("POST", "/$sourceIndex2,$targetIndex2/_search", emptyMap(), StringEntity(req, ContentType.APPLICATION_JSON)) assertTrue(searchResult1.restStatus() == RestStatus.OK) val failures = extractFailuresFromSearchResponse(searchResult1) @@ -1112,6 +1112,29 @@ class RollupInterceptorIT : RollupRestTestCase() { "Not all indices have rollup job", failures?.get(0)?.get("reason") ?: "Didn't find failure reason in search response", ) + // Updating to allow searching on non-rollup and rolled-up index together + updateSearchRawRollupClusterSetting(true) + val rawRes1 = client().makeRequest("POST", "/$sourceIndex2/_search", emptyMap(), StringEntity(req, ContentType.APPLICATION_JSON)) + assertTrue(rawRes1.restStatus() == RestStatus.OK) + val rawRes2 = client().makeRequest("POST", "/$targetIndex2/_search", emptyMap(), StringEntity(req, ContentType.APPLICATION_JSON)) + assertTrue(rawRes2.restStatus() == RestStatus.OK) + val searchResult = client().makeRequest("POST", "/$sourceIndex2,$targetIndex2/_search", emptyMap(), StringEntity(req, ContentType.APPLICATION_JSON)) + assertTrue(searchResult.restStatus() == RestStatus.OK) + val rawAgg1Res = rawRes1.asMap()["aggregations"] as Map> + val rawAgg2Res = rawRes2.asMap()["aggregations"] as Map> + val rollupAggResMulti = searchResult.asMap()["aggregations"] as Map> + + val trueAggSum = rawAgg1Res.getValue("sum_passenger_count")["value"] as Double + rawAgg2Res.getValue("sum_passenger_count")["value"] as Double + + assertEquals( + "Searching single raw source index and rollup target index did not return the same sum results", + rawAgg1Res.getValue("max_passenger_count")["value"], rollupAggResMulti.getValue("max_passenger_count")["value"], + ) + assertEquals( + "Searching rollup target index did not return the sum for all of the rollup jobs on the index", + trueAggSum, rollupAggResMulti.getValue("sum_passenger_count")["value"], + ) + // Search 2 rollups with different mappings try { client().makeRequest( From 94a23d29ff5203f76df3e450800d44f86b3b5d7e Mon Sep 17 00:00:00 2001 From: Kshitij Tandon Date: Tue, 29 Oct 2024 18:40:02 +0530 Subject: [PATCH 25/37] Added release notes for v2.18 (#1282) * Added release notes for v2.18 Signed-off-by: Kshitij Tandon * Adding another recent commit in the release notes Signed-off-by: Kshitij Tandon --------- Signed-off-by: Kshitij Tandon --- ...index-management.release-notes-2.18.0.0.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 release-notes/opensearch-index-management.release-notes-2.18.0.0.md diff --git a/release-notes/opensearch-index-management.release-notes-2.18.0.0.md b/release-notes/opensearch-index-management.release-notes-2.18.0.0.md new file mode 100644 index 000000000..290a6a8a9 --- /dev/null +++ b/release-notes/opensearch-index-management.release-notes-2.18.0.0.md @@ -0,0 +1,19 @@ +## Version 2.18.0.0 2024-10-28 + +Compatible with OpenSearch 2.18.0 + +### Enhancements + +* Allowing non-rollup and rollup indices to be searched together ([#1268](https://github.com/opensearch-project/index-management/pull/1268)) + +### Bug fixes +* Fixing snapshot bug ([#1257](https://github.com/opensearch-project/index-management/pull/1257)) + +### Maintenance +* Increment version to 2.18.0-SNAPSHOT ([#1241](https://github.com/opensearch-project/index-management/pull/1241)) +* upgrade upload-artifact to version 3 ([#1252](https://github.com/opensearch-project/index-management/pull/1252)) +* bump bwc version after 2.17 release ([#1259](https://github.com/opensearch-project/index-management/pull/1259)) +* Move non-active maintainer to emeritus ([#1263](https://github.com/opensearch-project/index-management/pull/1263)) +* Update CI check for integ-test-with-security to run all integ tests with security ([#1243](https://github.com/opensearch-project/index-management/pull/1243)) +* Remove 2 instances wildcard imports ([#1251](https://github.com/opensearch-project/index-management/pull/1251)) +* Updating baseline JDK version to JDK-21 ([#1276](https://github.com/opensearch-project/index-management/pull/1276)) \ No newline at end of file From ca2ada20d59be44d0cd10c38658eb9eb20c1d64c Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Tue, 29 Oct 2024 11:47:15 -0400 Subject: [PATCH 26/37] Use adminClient when searching system index in integ tests (#1286) * Use adminClient when searching system index in integ tests Signed-off-by: Craig Perkins * Replace more instances Signed-off-by: Craig Perkins * Another instance Signed-off-by: Craig Perkins --------- Signed-off-by: Craig Perkins --- .../IndexStateManagementRestTestCase.kt | 8 ++++---- .../resthandler/IndexStateManagementRestApiIT.kt | 4 ++-- .../indexmanagement/rollup/RollupRestTestCase.kt | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt index 1e37cdf4b..68c25a6d6 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt @@ -379,7 +379,7 @@ abstract class IndexStateManagementRestTestCase : IndexManagementRestTestCase() } """.trimIndent() val response = - client().makeRequest( + adminClient().makeRequest( "POST", "$INDEX_MANAGEMENT_INDEX/_search", emptyMap(), StringEntity(request, ContentType.APPLICATION_JSON), ) @@ -394,7 +394,7 @@ abstract class IndexStateManagementRestTestCase : IndexManagementRestTestCase() } protected fun getManagedIndexConfigByDocId(id: String): ManagedIndexConfig? { - val response = client().makeRequest("GET", "$INDEX_MANAGEMENT_INDEX/_doc/$id") + val response = adminClient().makeRequest("GET", "$INDEX_MANAGEMENT_INDEX/_doc/$id") assertEquals("Request failed", RestStatus.OK, response.restStatus()) val getResponse = GetResponse.fromXContent(createParser(jsonXContent, response.entity.content)) assertTrue("Did not find managed index config", getResponse.isExists) @@ -854,7 +854,7 @@ abstract class IndexStateManagementRestTestCase : IndexManagementRestTestCase() metadataId: String, header: BasicHeader = BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json"), ): RollupMetadata { - val response = client().makeRequest("GET", "$INDEX_MANAGEMENT_INDEX/_doc/$metadataId", null, header) + val response = adminClient().makeRequest("GET", "$INDEX_MANAGEMENT_INDEX/_doc/$metadataId", null, header) assertEquals("Unable to get rollup metadata $metadataId", RestStatus.OK, response.restStatus()) val parser = createParser(XContentType.JSON.xContent(), response.entity.content) @@ -911,7 +911,7 @@ abstract class IndexStateManagementRestTestCase : IndexManagementRestTestCase() metadataId: String, header: BasicHeader = BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json"), ): TransformMetadata { - val response = client().makeRequest("GET", "$INDEX_MANAGEMENT_INDEX/_doc/$metadataId", null, header) + val response = adminClient().makeRequest("GET", "$INDEX_MANAGEMENT_INDEX/_doc/$metadataId", null, header) assertEquals("Unable to get transform metadata $metadataId", RestStatus.OK, response.restStatus()) val parser = createParser(XContentType.JSON.xContent(), response.entity.content) diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/IndexStateManagementRestApiIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/IndexStateManagementRestApiIT.kt index a102d4e8e..f3ce9bc02 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/IndexStateManagementRestApiIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/IndexStateManagementRestApiIT.kt @@ -150,7 +150,7 @@ class IndexStateManagementRestApiIT : IndexStateManagementRestTestCase() { fun `test mappings after policy creation`() { createRandomPolicy() - val response = client().makeRequest("GET", "/$INDEX_MANAGEMENT_INDEX/_mapping") + val response = adminClient().makeRequest("GET", "/$INDEX_MANAGEMENT_INDEX/_mapping") val parserMap = createParser(XContentType.JSON.xContent(), response.entity.content).map() as Map> val mappingsMap = parserMap[INDEX_MANAGEMENT_INDEX]!!["mappings"] as Map @@ -271,7 +271,7 @@ class IndexStateManagementRestApiIT : IndexStateManagementRestTestCase() { } """.trimIndent() val response = - client().makeRequest( + adminClient().makeRequest( "POST", "$INDEX_MANAGEMENT_INDEX/_search", emptyMap(), StringEntity(request, ContentType.APPLICATION_JSON), ) diff --git a/src/test/kotlin/org/opensearch/indexmanagement/rollup/RollupRestTestCase.kt b/src/test/kotlin/org/opensearch/indexmanagement/rollup/RollupRestTestCase.kt index 6e527abfb..12e75b4e3 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/rollup/RollupRestTestCase.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/rollup/RollupRestTestCase.kt @@ -209,7 +209,7 @@ abstract class RollupRestTestCase : IndexManagementRestTestCase() { refresh: Boolean = true, header: BasicHeader = BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json"), ): RollupMetadata { - val response = client().makeRequest("GET", "$INDEX_MANAGEMENT_INDEX/_doc/$metadataId?refresh=$refresh", null, header) + val response = adminClient().makeRequest("GET", "$INDEX_MANAGEMENT_INDEX/_doc/$metadataId?refresh=$refresh", null, header) assertEquals("Unable to get rollup metadata $metadataId", RestStatus.OK, response.restStatus()) return parseRollupMetadata(response) } @@ -220,7 +220,7 @@ abstract class RollupRestTestCase : IndexManagementRestTestCase() { refresh: Boolean = true, header: BasicHeader = BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json"), ): RollupMetadata { - val response = client().makeRequest("GET", "$INDEX_MANAGEMENT_INDEX/_doc/$metadataId?routing=$routingId&refresh=$refresh", null, header) + val response = adminClient().makeRequest("GET", "$INDEX_MANAGEMENT_INDEX/_doc/$metadataId?routing=$routingId&refresh=$refresh", null, header) assertEquals("Unable to get rollup metadata $metadataId", RestStatus.OK, response.restStatus()) return parseRollupMetadata(response) From 5aaf114992f70173f232309d5bb9796c11e17c9a Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Tue, 29 Oct 2024 18:44:35 -0400 Subject: [PATCH 27/37] Add dependabot.yml to monitor outdated dependencies (#1289) --- .github/dependabot.yml | 18 ++++++++++++++++++ build.gradle | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..9088426cf --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,18 @@ +version: 2 +updates: + - package-ecosystem: "gradle" + directory: "/" + schedule: + interval: "weekly" + commit-message: + prefix: "dependabot:" + ignore: + # For all packages, ignore all major versions to minimize breaking issues + - dependency-name: "*" + update-types: ["version-update:semver-major"] + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + commit-message: + prefix: "dependabot:" \ No newline at end of file diff --git a/build.gradle b/build.gradle index 5ab7defc4..257c51520 100644 --- a/build.gradle +++ b/build.gradle @@ -638,7 +638,7 @@ task integTestRemote(type: RestIntegTestTask) { // === Set up BWC tests === -String bwcVersionShort = "2.18.0" +String bwcVersionShort = "2.19.0" String bwcVersion = bwcVersionShort + ".0" String baseName = "indexmanagementBwcCluster" From 6c263f9205d0bd24751c73ba0513ab43dddfe2c6 Mon Sep 17 00:00:00 2001 From: Kshitij Tandon Date: Mon, 9 Dec 2024 12:35:44 +0530 Subject: [PATCH 28/37] Adding maintainers for the index-management ISM repo (#1314) * Adding maintainers for the index-management ISM repo Signed-off-by: Kshitij Tandon * Adding maintainers to the CODEOWNERS file also Signed-off-by: Kshitij Tandon --------- Signed-off-by: Kshitij Tandon --- .github/CODEOWNERS | 2 +- MAINTAINERS.md | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index cd9176ea7..8926d78d5 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -* @bowenlan-amzn @Hailong-am @vikasvb90 +* @bowenlan-amzn @Hailong-am @vikasvb90 @tandonks @shiv0408 @soosinha diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 8127dcfa5..047e019f8 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -9,6 +9,9 @@ This document contains a list of maintainers in this repo. See [opensearch-proje | Vikas Bansal | [vikasvb90](https://github.com/vikasvb90) | Amazon | | Bowen Lan | [bowenlan-amzn](https://github.com/bowenlan-amzn) | Amazon | | Hailong Cui | [Hailong-am](https://github.com/Hailong-am) | Amazon | +| Kshitij Tandon | [tandonks](https://github.com/tandonks) | Amazon | +| Shivansh Arora | [shiv0408](https://github.com/shiv0408) | Amazon | +| Sooraj Sinha | [soosinha](https://github.com/soosinha) | Amazon | ## Emeritus From e83ad22b8a505068e22f1aab42e91eeaf5ad8498 Mon Sep 17 00:00:00 2001 From: Kshitij Tandon Date: Wed, 11 Dec 2024 13:50:13 +0530 Subject: [PATCH 29/37] Revert "Adding maintainers for the index-management ISM repo (#1314)" (#1319) This reverts commit 6c263f9205d0bd24751c73ba0513ab43dddfe2c6. Signed-off-by: Kshitij Tandon --- .github/CODEOWNERS | 2 +- MAINTAINERS.md | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 8926d78d5..cd9176ea7 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -* @bowenlan-amzn @Hailong-am @vikasvb90 @tandonks @shiv0408 @soosinha +* @bowenlan-amzn @Hailong-am @vikasvb90 diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 047e019f8..8127dcfa5 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -9,9 +9,6 @@ This document contains a list of maintainers in this repo. See [opensearch-proje | Vikas Bansal | [vikasvb90](https://github.com/vikasvb90) | Amazon | | Bowen Lan | [bowenlan-amzn](https://github.com/bowenlan-amzn) | Amazon | | Hailong Cui | [Hailong-am](https://github.com/Hailong-am) | Amazon | -| Kshitij Tandon | [tandonks](https://github.com/tandonks) | Amazon | -| Shivansh Arora | [shiv0408](https://github.com/shiv0408) | Amazon | -| Sooraj Sinha | [soosinha](https://github.com/soosinha) | Amazon | ## Emeritus From 6617cf047e6dab0f61ac9752f2d6bfbca836db1e Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Fri, 13 Dec 2024 12:15:53 -0500 Subject: [PATCH 30/37] Make ActionRequests that extend UpdateRequest to extend more generic ActionRequest (#1311) * Upgrade to actions/upload-artifact@v4 Signed-off-by: Craig Perkins * Make ActionRequests that extend UpdateRequest to extend more generic ActionRequest Signed-off-by: Craig Perkins * Apply to StartTransformRequest Signed-off-by: Craig Perkins * Apply to rollup actions Signed-off-by: Craig Perkins * Update snapshot management requests Signed-off-by: Craig Perkins * Fix tests Signed-off-by: Craig Perkins * Free up disk space on github runner Signed-off-by: Craig Perkins * Remove sudo Signed-off-by: Craig Perkins * Check runner space Signed-off-by: Craig Perkins * Remove sudo Signed-off-by: Craig Perkins * Delete index in test Signed-off-by: Craig Perkins * Skip generic Signed-off-by: Craig Perkins * Clear AfterClass Signed-off-by: Craig Perkins * Permissive warnings handler Signed-off-by: Craig Perkins * Move into waitFor Signed-off-by: Craig Perkins * Remove match to generic Signed-off-by: Craig Perkins * Revert "Permissive warnings handler" This reverts commit 7c824704dd3d06ad5e917a12859fa49647d7b46f. Signed-off-by: Craig Perkins * Move back out of waitFor Signed-off-by: Craig Perkins * Fix issue with readonly indices after bugfix in core (https://github.com/opensearch-project/OpenSearch/pull/16568) Signed-off-by: Craig Perkins * Overwrite true Signed-off-by: Craig Perkins --------- Signed-off-by: Craig Perkins --- .../workflows/multi-node-test-workflow.yml | 13 ++++++------ .github/workflows/security-test-workflow.yml | 13 ++++++------ .github/workflows/test-and-build-workflow.yml | 20 +++++++++--------- .../TransportRemovePolicyAction.kt | 18 ++++++++++++---- .../rollup/action/start/StartRollupRequest.kt | 17 ++++++++++----- .../start/TransportStartRollupAction.kt | 8 ++++--- .../rollup/action/stop/StopRollupRequest.kt | 17 ++++++++++----- .../action/stop/TransportStopRollupAction.kt | 9 ++++---- .../api/transport/start/StartSMRequest.kt | 21 ++++++++++++------- .../transport/start/TransportStartSMAction.kt | 12 ++++++----- .../api/transport/stop/StopSMRequest.kt | 21 ++++++++++++------- .../transport/stop/TransportStopSMAction.kt | 12 ++++++----- .../action/start/StartTransformRequest.kt | 17 ++++++++++----- .../start/TransportStartTransformAction.kt | 7 ++++--- .../action/stop/StopTransformRequest.kt | 17 ++++++++++----- .../stop/TransportStopTransformAction.kt | 9 ++++---- .../rollup/action/RequestTests.kt | 8 +++---- .../snapshotmanagement/action/RequestTests.kt | 8 +++---- .../transform/action/RequestTests.kt | 8 +++---- 19 files changed, 157 insertions(+), 98 deletions(-) diff --git a/.github/workflows/multi-node-test-workflow.yml b/.github/workflows/multi-node-test-workflow.yml index 40d9cc905..02a693ff1 100644 --- a/.github/workflows/multi-node-test-workflow.yml +++ b/.github/workflows/multi-node-test-workflow.yml @@ -7,8 +7,6 @@ on: push: branches: - "**" -env: - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true jobs: Get-CI-Image-Tag: @@ -35,25 +33,26 @@ jobs: # using the same image which is used by opensearch-build team to build the OpenSearch Distribution # this image tag is subject to change as more dependencies and updates will arrive over time image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} - # need to switch to root so that github actions can install runner binary on container without permission issues. - options: --user root + options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} steps: + - name: Run start commands + run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }} # This step uses the setup-java Github action: https://github.com/actions/setup-java - name: Set Up JDK - uses: actions/setup-java@v2 + uses: actions/setup-java@v4 with: distribution: temurin # Temurin is a distribution of adoptium java-version: 21 # index-management - name: Checkout Branch - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Run integration tests with multi node config run: | chown -R 1000:1000 `pwd` su `id -un 1000` -c "./gradlew integTest -PnumNodes=3 ${{ env.TEST_FILTER }}" - name: Upload failed logs - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: failure() with: name: logs diff --git a/.github/workflows/security-test-workflow.yml b/.github/workflows/security-test-workflow.yml index 8dc6eb19c..6b0363fc7 100644 --- a/.github/workflows/security-test-workflow.yml +++ b/.github/workflows/security-test-workflow.yml @@ -7,8 +7,6 @@ on: push: branches: - "**" -env: - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true jobs: Get-CI-Image-Tag: @@ -24,25 +22,26 @@ jobs: # using the same image which is used by opensearch-build team to build the OpenSearch Distribution # this image tag is subject to change as more dependencies and updates will arrive over time image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} - # need to switch to root so that github actions can install runner binary on container without permission issues. - options: --user root + options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} steps: + - name: Run start commands + run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }} # This step uses the setup-java Github action: https://github.com/actions/setup-java - name: Set Up JDK - uses: actions/setup-java@v2 + uses: actions/setup-java@v4 with: distribution: temurin # Temurin is a distribution of adoptium java-version: 21 # index-management - name: Checkout Branch - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Run integration tests run: | chown -R 1000:1000 `pwd` su `id -un 1000` -c "./gradlew integTest -Dsecurity=true -Dhttps=true --tests '*IT'" - name: Upload failed logs - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: failure() with: name: logs diff --git a/.github/workflows/test-and-build-workflow.yml b/.github/workflows/test-and-build-workflow.yml index 5713aea5e..32870eefa 100644 --- a/.github/workflows/test-and-build-workflow.yml +++ b/.github/workflows/test-and-build-workflow.yml @@ -6,8 +6,6 @@ on: push: branches: - "**" -env: - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true jobs: Get-CI-Image-Tag: @@ -36,19 +34,20 @@ jobs: # using the same image which is used by opensearch-build team to build the OpenSearch Distribution # this image tag is subject to change as more dependencies and updates will arrive over time image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} - # need to switch to root so that github actions can install runner binary on container without permission issues. - options: --user root + options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} steps: + - name: Run start commands + run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }} # This step uses the setup-java Github action: https://github.com/actions/setup-java - name: Set Up JDK ${{ matrix.java }} - uses: actions/setup-java@v2 + uses: actions/setup-java@v4 with: distribution: temurin # Temurin is a distribution of adoptium java-version: ${{ matrix.java }} # build index management - name: Checkout Branch - uses: actions/checkout@v2 + uses: actions/checkout@v4 # This is a hack, but this step creates a link to the X: mounted drive, which makes the path # short enough to work on Windows - name: Build with Gradle @@ -61,18 +60,19 @@ jobs: with: name: logs-${{ matrix.java }}-${{ matrix.feature }} path: build/testclusters/integTest-*/logs/* + overwrite: 'true' - name: Create Artifact Path run: | mkdir -p index-management-artifacts cp ./build/distributions/*.zip index-management-artifacts - name: Uploads coverage - uses: codecov/codecov-action@v1 + uses: codecov/codecov-action@v4 with: token: ${{ secrets.CODECOV_TOKEN }} # This step uses the upload-artifact Github action: https://github.com/actions/upload-artifact - name: Upload Artifacts # v4 requires node.js 20 which is not supported - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: index-management-plugin-ubuntu-latest-${{ matrix.java }} path: index-management-artifacts @@ -106,13 +106,13 @@ jobs: steps: # This step uses the setup-java Github action: https://github.com/actions/setup-java - name: Set Up JDK ${{ matrix.java }} - uses: actions/setup-java@v2 + uses: actions/setup-java@v4 with: distribution: temurin # Temurin is a distribution of adoptium java-version: ${{ matrix.java }} # build index management - name: Checkout Branch - uses: actions/checkout@v2 + uses: actions/checkout@v4 # This is a hack, but this step creates a link to the X: mounted drive, which makes the path # short enough to work on Windows - name: Shorten Path diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/removepolicy/TransportRemovePolicyAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/removepolicy/TransportRemovePolicyAction.kt index dcefa694e..55150225e 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/removepolicy/TransportRemovePolicyAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/removepolicy/TransportRemovePolicyAction.kt @@ -268,8 +268,13 @@ constructor( updateSettingReqsList.add( UpdateSettingsRequest().indices(*readOnlyIndices.map { indices[it] }.toTypedArray()) .settings( - Settings.builder().put(ManagedIndexSettings.AUTO_MANAGE.key, false) - .put(INDEX_READ_ONLY_SETTING.key, true), + Settings.builder().put(INDEX_READ_ONLY_SETTING.key, false), + ), + ) + updateSettingReqsList.add( + UpdateSettingsRequest().indices(*readOnlyIndices.map { indices[it] }.toTypedArray()) + .settings( + Settings.builder().put(ManagedIndexSettings.AUTO_MANAGE.key, false).put(INDEX_READ_ONLY_SETTING.key, true), ), ) } @@ -277,8 +282,13 @@ constructor( updateSettingReqsList.add( UpdateSettingsRequest().indices(*readOnlyAllowDeleteIndices.map { indices[it] }.toTypedArray()) .settings( - Settings.builder().put(ManagedIndexSettings.AUTO_MANAGE.key, false) - .put(INDEX_BLOCKS_READ_ONLY_ALLOW_DELETE_SETTING.key, true), + Settings.builder().put(INDEX_BLOCKS_READ_ONLY_ALLOW_DELETE_SETTING.key, false), + ), + ) + updateSettingReqsList.add( + UpdateSettingsRequest().indices(*readOnlyAllowDeleteIndices.map { indices[it] }.toTypedArray()) + .settings( + Settings.builder().put(ManagedIndexSettings.AUTO_MANAGE.key, false).put(INDEX_BLOCKS_READ_ONLY_ALLOW_DELETE_SETTING.key, true), ), ) } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/rollup/action/start/StartRollupRequest.kt b/src/main/kotlin/org/opensearch/indexmanagement/rollup/action/start/StartRollupRequest.kt index 3493d5473..5d4e63b36 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/rollup/action/start/StartRollupRequest.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/rollup/action/start/StartRollupRequest.kt @@ -5,24 +5,30 @@ package org.opensearch.indexmanagement.rollup.action.start +import org.opensearch.action.ActionRequest import org.opensearch.action.ActionRequestValidationException import org.opensearch.action.ValidateActions.addValidationError -import org.opensearch.action.update.UpdateRequest import org.opensearch.core.common.io.stream.StreamInput import org.opensearch.core.common.io.stream.StreamOutput import java.io.IOException -class StartRollupRequest : UpdateRequest { +class StartRollupRequest : ActionRequest { + + val id: String + get() = field + @Throws(IOException::class) - constructor(sin: StreamInput) : super(sin) + constructor(sin: StreamInput) : super(sin) { + this.id = sin.readString() + } constructor(id: String) { - super.id(id) + this.id = id } override fun validate(): ActionRequestValidationException? { var validationException: ActionRequestValidationException? = null - if (super.id().isEmpty()) { + if (this.id.isEmpty()) { validationException = addValidationError("id is missing", validationException) } return validationException @@ -31,5 +37,6 @@ class StartRollupRequest : UpdateRequest { @Throws(IOException::class) override fun writeTo(out: StreamOutput) { super.writeTo(out) + out.writeString(id) } } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/rollup/action/start/TransportStartRollupAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/rollup/action/start/TransportStartRollupAction.kt index 535c834b5..8689dc221 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/rollup/action/start/TransportStartRollupAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/rollup/action/start/TransportStartRollupAction.kt @@ -28,6 +28,7 @@ import org.opensearch.commons.authuser.User import org.opensearch.core.action.ActionListener import org.opensearch.core.rest.RestStatus import org.opensearch.core.xcontent.NamedXContentRegistry +import org.opensearch.indexmanagement.IndexManagementPlugin import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX import org.opensearch.indexmanagement.opensearchapi.parseWithType import org.opensearch.indexmanagement.rollup.model.Rollup @@ -70,7 +71,7 @@ constructor( ConfigConstants.OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT, )}", ) - val getReq = GetRequest(INDEX_MANAGEMENT_INDEX, request.id()) + val getReq = GetRequest(INDEX_MANAGEMENT_INDEX, request.id) val user: User? = buildUser(client.threadPool().threadContext) client.threadPool().threadContext.stashContext().use { client.get( @@ -115,7 +116,8 @@ constructor( // TODO: Should create a transport action to update metadata private fun updateRollupJob(rollup: Rollup, request: StartRollupRequest, actionListener: ActionListener) { val now = Instant.now().toEpochMilli() - request.index(INDEX_MANAGEMENT_INDEX).doc( + val updateReq = UpdateRequest(IndexManagementPlugin.INDEX_MANAGEMENT_INDEX, request.id) + updateReq.doc( mapOf( Rollup.ROLLUP_TYPE to mapOf( @@ -125,7 +127,7 @@ constructor( ), ) client.update( - request, + updateReq, object : ActionListener { override fun onResponse(response: UpdateResponse) { if (response.result == DocWriteResponse.Result.UPDATED) { diff --git a/src/main/kotlin/org/opensearch/indexmanagement/rollup/action/stop/StopRollupRequest.kt b/src/main/kotlin/org/opensearch/indexmanagement/rollup/action/stop/StopRollupRequest.kt index 1a03317a7..0b5d4b75d 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/rollup/action/stop/StopRollupRequest.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/rollup/action/stop/StopRollupRequest.kt @@ -5,24 +5,30 @@ package org.opensearch.indexmanagement.rollup.action.stop +import org.opensearch.action.ActionRequest import org.opensearch.action.ActionRequestValidationException import org.opensearch.action.ValidateActions.addValidationError -import org.opensearch.action.update.UpdateRequest import org.opensearch.core.common.io.stream.StreamInput import org.opensearch.core.common.io.stream.StreamOutput import java.io.IOException -class StopRollupRequest : UpdateRequest { +class StopRollupRequest : ActionRequest { + + val id: String + get() = field + @Throws(IOException::class) - constructor(sin: StreamInput) : super(sin) + constructor(sin: StreamInput) : super(sin) { + this.id = sin.readString() + } constructor(id: String) { - super.id(id) + this.id = id } override fun validate(): ActionRequestValidationException? { var validationException: ActionRequestValidationException? = null - if (super.id().isEmpty()) { + if (this.id.isEmpty()) { validationException = addValidationError("id is missing", validationException) } return validationException @@ -31,5 +37,6 @@ class StopRollupRequest : UpdateRequest { @Throws(IOException::class) override fun writeTo(out: StreamOutput) { super.writeTo(out) + out.writeString(id) } } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/rollup/action/stop/TransportStopRollupAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/rollup/action/stop/TransportStopRollupAction.kt index fdeead257..8aad70107 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/rollup/action/stop/TransportStopRollupAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/rollup/action/stop/TransportStopRollupAction.kt @@ -77,13 +77,13 @@ constructor( @Suppress("ReturnCount") override fun doExecute(task: Task, request: StopRollupRequest, actionListener: ActionListener) { - log.debug("Executing StopRollupAction on ${request.id()}") + log.debug("Executing StopRollupAction on ${request.id}") log.debug( "User and roles string from thread context: ${client.threadPool().threadContext.getTransient( ConfigConstants.OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT, )}", ) - val getRequest = GetRequest(IndexManagementPlugin.INDEX_MANAGEMENT_INDEX, request.id()) + val getRequest = GetRequest(IndexManagementPlugin.INDEX_MANAGEMENT_INDEX, request.id) val user = buildUser(client.threadPool().threadContext) client.threadPool().threadContext.stashContext().use { client.get( @@ -214,7 +214,8 @@ constructor( private fun updateRollupJob(rollup: Rollup, request: StopRollupRequest, actionListener: ActionListener) { val now = Instant.now().toEpochMilli() - request.index(IndexManagementPlugin.INDEX_MANAGEMENT_INDEX).setIfSeqNo(rollup.seqNo).setIfPrimaryTerm(rollup.primaryTerm) + val updateReq = UpdateRequest(IndexManagementPlugin.INDEX_MANAGEMENT_INDEX, request.id) + updateReq.setIfSeqNo(rollup.seqNo).setIfPrimaryTerm(rollup.primaryTerm) .doc( mapOf( Rollup.ROLLUP_TYPE to @@ -226,7 +227,7 @@ constructor( ) .routing(rollup.id) client.update( - request, + updateReq, object : ActionListener { override fun onResponse(response: UpdateResponse) { actionListener.onResponse(AcknowledgedResponse(response.result == DocWriteResponse.Result.UPDATED)) diff --git a/src/main/kotlin/org/opensearch/indexmanagement/snapshotmanagement/api/transport/start/StartSMRequest.kt b/src/main/kotlin/org/opensearch/indexmanagement/snapshotmanagement/api/transport/start/StartSMRequest.kt index 23e00e16b..6b22dd99e 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/snapshotmanagement/api/transport/start/StartSMRequest.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/snapshotmanagement/api/transport/start/StartSMRequest.kt @@ -5,25 +5,31 @@ package org.opensearch.indexmanagement.snapshotmanagement.api.transport.start +import org.opensearch.action.ActionRequest import org.opensearch.action.ActionRequestValidationException -import org.opensearch.action.ValidateActions -import org.opensearch.action.update.UpdateRequest +import org.opensearch.action.ValidateActions.addValidationError import org.opensearch.core.common.io.stream.StreamInput import org.opensearch.core.common.io.stream.StreamOutput import java.io.IOException -class StartSMRequest : UpdateRequest { +class StartSMRequest : ActionRequest { + + val id: String + get() = field + @Throws(IOException::class) - constructor(sin: StreamInput) : super(sin) + constructor(sin: StreamInput) : super(sin) { + this.id = sin.readString() + } constructor(id: String) { - super.id(id) + this.id = id } override fun validate(): ActionRequestValidationException? { var validationException: ActionRequestValidationException? = null - if (super.id().isEmpty()) { - validationException = ValidateActions.addValidationError("id is missing", validationException) + if (this.id.isEmpty()) { + validationException = addValidationError("id is missing", validationException) } return validationException } @@ -31,5 +37,6 @@ class StartSMRequest : UpdateRequest { @Throws(IOException::class) override fun writeTo(out: StreamOutput) { super.writeTo(out) + out.writeString(id) } } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/snapshotmanagement/api/transport/start/TransportStartSMAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/snapshotmanagement/api/transport/start/TransportStartSMAction.kt index ac5e58ab5..6e81bb6b7 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/snapshotmanagement/api/transport/start/TransportStartSMAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/snapshotmanagement/api/transport/start/TransportStartSMAction.kt @@ -11,6 +11,7 @@ import org.opensearch.OpenSearchStatusException import org.opensearch.action.DocWriteResponse import org.opensearch.action.support.ActionFilters import org.opensearch.action.support.master.AcknowledgedResponse +import org.opensearch.action.update.UpdateRequest import org.opensearch.action.update.UpdateResponse import org.opensearch.client.Client import org.opensearch.cluster.service.ClusterService @@ -57,7 +58,7 @@ constructor( user: User?, threadContext: ThreadContext.StoredContext, ): AcknowledgedResponse { - val smPolicy = client.getSMPolicy(request.id()) + val smPolicy = client.getSMPolicy(request.id) // Check if the requested user has permission on the resource, throwing an exception if the user does not verifyUserHasPermissionForResource(user, smPolicy.user, filterByEnabled, "snapshot management policy", smPolicy.policyName) @@ -71,7 +72,8 @@ constructor( private suspend fun enableSMPolicy(updateRequest: StartSMRequest): Boolean { val now = Instant.now().toEpochMilli() - updateRequest.index(INDEX_MANAGEMENT_INDEX).doc( + val updateReq = UpdateRequest(INDEX_MANAGEMENT_INDEX, updateRequest.id) + updateReq.doc( mapOf( SMPolicy.SM_TYPE to mapOf( @@ -83,12 +85,12 @@ constructor( ) val updateResponse: UpdateResponse = try { - client.suspendUntil { update(updateRequest, it) } + client.suspendUntil { update(updateReq, it) } } catch (e: VersionConflictEngineException) { - log.error("VersionConflictEngineException while trying to enable snapshot management policy id [${updateRequest.id()}]: $e") + log.error("VersionConflictEngineException while trying to enable snapshot management policy id [${updateRequest.id}]: $e") throw OpenSearchStatusException(conflictExceptionMessage, RestStatus.INTERNAL_SERVER_ERROR) } catch (e: Exception) { - log.error("Failed trying to enable snapshot management policy id [${updateRequest.id()}]: $e") + log.error("Failed trying to enable snapshot management policy id [${updateRequest.id}]: $e") throw OpenSearchStatusException("Failed while trying to enable SM Policy", RestStatus.INTERNAL_SERVER_ERROR) } return updateResponse.result == DocWriteResponse.Result.UPDATED diff --git a/src/main/kotlin/org/opensearch/indexmanagement/snapshotmanagement/api/transport/stop/StopSMRequest.kt b/src/main/kotlin/org/opensearch/indexmanagement/snapshotmanagement/api/transport/stop/StopSMRequest.kt index 4d70096b6..1b7961e09 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/snapshotmanagement/api/transport/stop/StopSMRequest.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/snapshotmanagement/api/transport/stop/StopSMRequest.kt @@ -5,25 +5,31 @@ package org.opensearch.indexmanagement.snapshotmanagement.api.transport.stop +import org.opensearch.action.ActionRequest import org.opensearch.action.ActionRequestValidationException -import org.opensearch.action.ValidateActions -import org.opensearch.action.update.UpdateRequest +import org.opensearch.action.ValidateActions.addValidationError import org.opensearch.core.common.io.stream.StreamInput import org.opensearch.core.common.io.stream.StreamOutput import java.io.IOException -class StopSMRequest : UpdateRequest { +class StopSMRequest : ActionRequest { + + val id: String + get() = field + @Throws(IOException::class) - constructor(sin: StreamInput) : super(sin) + constructor(sin: StreamInput) : super(sin) { + this.id = sin.readString() + } constructor(id: String) { - super.id(id) + this.id = id } override fun validate(): ActionRequestValidationException? { var validationException: ActionRequestValidationException? = null - if (super.id().isEmpty()) { - validationException = ValidateActions.addValidationError("id is missing", validationException) + if (this.id.isEmpty()) { + validationException = addValidationError("id is missing", validationException) } return validationException } @@ -31,5 +37,6 @@ class StopSMRequest : UpdateRequest { @Throws(IOException::class) override fun writeTo(out: StreamOutput) { super.writeTo(out) + out.writeString(id) } } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/snapshotmanagement/api/transport/stop/TransportStopSMAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/snapshotmanagement/api/transport/stop/TransportStopSMAction.kt index 4e4b0b7e6..6d45735d6 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/snapshotmanagement/api/transport/stop/TransportStopSMAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/snapshotmanagement/api/transport/stop/TransportStopSMAction.kt @@ -11,6 +11,7 @@ import org.opensearch.OpenSearchStatusException import org.opensearch.action.DocWriteResponse import org.opensearch.action.support.ActionFilters import org.opensearch.action.support.master.AcknowledgedResponse +import org.opensearch.action.update.UpdateRequest import org.opensearch.action.update.UpdateResponse import org.opensearch.client.Client import org.opensearch.cluster.service.ClusterService @@ -57,7 +58,7 @@ constructor( user: User?, threadContext: ThreadContext.StoredContext, ): AcknowledgedResponse { - val smPolicy = client.getSMPolicy(request.id()) + val smPolicy = client.getSMPolicy(request.id) // Check if the requested user has permission on the resource, throwing an exception if the user does not verifyUserHasPermissionForResource(user, smPolicy.user, filterByEnabled, "snapshot management policy", smPolicy.policyName) @@ -71,7 +72,8 @@ constructor( private suspend fun disableSMPolicy(updateRequest: StopSMRequest): Boolean { val now = Instant.now().toEpochMilli() - updateRequest.index(INDEX_MANAGEMENT_INDEX).doc( + val updateReq = UpdateRequest(INDEX_MANAGEMENT_INDEX, updateRequest.id) + updateReq.doc( mapOf( SMPolicy.SM_TYPE to mapOf( @@ -83,12 +85,12 @@ constructor( ) val updateResponse: UpdateResponse = try { - client.suspendUntil { update(updateRequest, it) } + client.suspendUntil { update(updateReq, it) } } catch (e: VersionConflictEngineException) { - log.error("VersionConflictEngineException while trying to disable snapshot management policy id [${updateRequest.id()}]: $e") + log.error("VersionConflictEngineException while trying to disable snapshot management policy id [${updateRequest.id}]: $e") throw OpenSearchStatusException(conflictExceptionMessage, RestStatus.INTERNAL_SERVER_ERROR) } catch (e: Exception) { - log.error("Failed trying to disable snapshot management policy id [${updateRequest.id()}]: $e") + log.error("Failed trying to disable snapshot management policy id [${updateRequest.id}]: $e") throw OpenSearchStatusException("Failed while trying to disable SM Policy", RestStatus.INTERNAL_SERVER_ERROR) } // TODO update metadata diff --git a/src/main/kotlin/org/opensearch/indexmanagement/transform/action/start/StartTransformRequest.kt b/src/main/kotlin/org/opensearch/indexmanagement/transform/action/start/StartTransformRequest.kt index a6514c6df..b9335c55c 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/transform/action/start/StartTransformRequest.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/transform/action/start/StartTransformRequest.kt @@ -5,24 +5,30 @@ package org.opensearch.indexmanagement.transform.action.start +import org.opensearch.action.ActionRequest import org.opensearch.action.ActionRequestValidationException import org.opensearch.action.ValidateActions.addValidationError -import org.opensearch.action.update.UpdateRequest import org.opensearch.core.common.io.stream.StreamInput import org.opensearch.core.common.io.stream.StreamOutput import java.io.IOException -class StartTransformRequest : UpdateRequest { +class StartTransformRequest : ActionRequest { + + val id: String + get() = field + @Throws(IOException::class) - constructor(sin: StreamInput) : super(sin) + constructor(sin: StreamInput) : super(sin) { + this.id = sin.readString() + } constructor(id: String) { - super.id(id) + this.id = id } override fun validate(): ActionRequestValidationException? { var validationException: ActionRequestValidationException? = null - if (super.id().isEmpty()) { + if (this.id.isEmpty()) { validationException = addValidationError("id is missing", validationException) } return validationException @@ -31,5 +37,6 @@ class StartTransformRequest : UpdateRequest { @Throws(IOException::class) override fun writeTo(out: StreamOutput) { super.writeTo(out) + out.writeString(id) } } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/transform/action/start/TransportStartTransformAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/transform/action/start/TransportStartTransformAction.kt index b69187d89..3a2c21044 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/transform/action/start/TransportStartTransformAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/transform/action/start/TransportStartTransformAction.kt @@ -68,7 +68,7 @@ constructor( ConfigConstants.OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT, )}", ) - val getRequest = GetRequest(INDEX_MANAGEMENT_INDEX, request.id()) + val getRequest = GetRequest(INDEX_MANAGEMENT_INDEX, request.id) val user = buildUser(client.threadPool().threadContext) client.threadPool().threadContext.stashContext().use { client.get( @@ -117,7 +117,8 @@ constructor( actionListener: ActionListener, ) { val now = Instant.now().toEpochMilli() - request.index(INDEX_MANAGEMENT_INDEX).doc( + val updateReq = UpdateRequest(INDEX_MANAGEMENT_INDEX, request.id) + updateReq.doc( mapOf( Transform.TRANSFORM_TYPE to mapOf( @@ -127,7 +128,7 @@ constructor( ), ) client.update( - request, + updateReq, object : ActionListener { override fun onResponse(response: UpdateResponse) { if (response.result == DocWriteResponse.Result.UPDATED) { diff --git a/src/main/kotlin/org/opensearch/indexmanagement/transform/action/stop/StopTransformRequest.kt b/src/main/kotlin/org/opensearch/indexmanagement/transform/action/stop/StopTransformRequest.kt index 7d3a3c2fc..5174abda3 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/transform/action/stop/StopTransformRequest.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/transform/action/stop/StopTransformRequest.kt @@ -5,24 +5,30 @@ package org.opensearch.indexmanagement.transform.action.stop +import org.opensearch.action.ActionRequest import org.opensearch.action.ActionRequestValidationException import org.opensearch.action.ValidateActions.addValidationError -import org.opensearch.action.update.UpdateRequest import org.opensearch.core.common.io.stream.StreamInput import org.opensearch.core.common.io.stream.StreamOutput import java.io.IOException -class StopTransformRequest : UpdateRequest { +class StopTransformRequest : ActionRequest { + + val id: String + get() = field + @Throws(IOException::class) - constructor(sin: StreamInput) : super(sin) + constructor(sin: StreamInput) : super(sin) { + this.id = sin.readString() + } constructor(id: String) { - super.id(id) + this.id = id } override fun validate(): ActionRequestValidationException? { var validationException: ActionRequestValidationException? = null - if (super.id().isEmpty()) { + if (this.id.isEmpty()) { validationException = addValidationError("id is missing", validationException) } return validationException @@ -31,5 +37,6 @@ class StopTransformRequest : UpdateRequest { @Throws(IOException::class) override fun writeTo(out: StreamOutput) { super.writeTo(out) + out.writeString(id) } } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/transform/action/stop/TransportStopTransformAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/transform/action/stop/TransportStopTransformAction.kt index 4e0e480cb..c2b5636d6 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/transform/action/stop/TransportStopTransformAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/transform/action/stop/TransportStopTransformAction.kt @@ -76,13 +76,13 @@ constructor( private val log = LogManager.getLogger(javaClass) override fun doExecute(task: Task, request: StopTransformRequest, actionListener: ActionListener) { - log.debug("Executing StopTransformAction on ${request.id()}") + log.debug("Executing StopTransformAction on ${request.id}") log.debug( "User and roles string from thread context: ${client.threadPool().threadContext.getTransient( ConfigConstants.OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT, )}", ) - val getRequest = GetRequest(INDEX_MANAGEMENT_INDEX, request.id()) + val getRequest = GetRequest(INDEX_MANAGEMENT_INDEX, request.id) val user = buildUser(client.threadPool().threadContext) client.threadPool().threadContext.stashContext().use { client.get( @@ -212,7 +212,8 @@ constructor( private fun updateTransformJob(transform: Transform, request: StopTransformRequest, actionListener: ActionListener) { val now = Instant.now().toEpochMilli() - request.index(IndexManagementPlugin.INDEX_MANAGEMENT_INDEX).setIfSeqNo(transform.seqNo).setIfPrimaryTerm(transform.primaryTerm) + val updateReq = UpdateRequest(IndexManagementPlugin.INDEX_MANAGEMENT_INDEX, request.id) + updateReq.setIfSeqNo(transform.seqNo).setIfPrimaryTerm(transform.primaryTerm) .doc( mapOf( Transform.TRANSFORM_TYPE to @@ -223,7 +224,7 @@ constructor( ), ) client.update( - request, + updateReq, object : ActionListener { override fun onResponse(response: UpdateResponse) { actionListener.onResponse(AcknowledgedResponse(response.result == DocWriteResponse.Result.UPDATED)) diff --git a/src/test/kotlin/org/opensearch/indexmanagement/rollup/action/RequestTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/rollup/action/RequestTests.kt index 3786a20f2..4b880336a 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/rollup/action/RequestTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/rollup/action/RequestTests.kt @@ -134,21 +134,21 @@ class RequestTests : OpenSearchTestCase() { fun `test start rollup request`() { val id = "some_id" - val req = StartRollupRequest(id).index(INDEX_MANAGEMENT_INDEX) + val req = StartRollupRequest(id) val out = BytesStreamOutput().apply { req.writeTo(this) } val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) val streamedReq = StartRollupRequest(sin) - assertEquals(id, streamedReq.id()) + assertEquals(id, streamedReq.id) } fun `test stop rollup request`() { val id = "some_id" - val req = StopRollupRequest(id).index(INDEX_MANAGEMENT_INDEX) + val req = StopRollupRequest(id) val out = BytesStreamOutput().apply { req.writeTo(this) } val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) val streamedReq = StopRollupRequest(sin) - assertEquals(id, streamedReq.id()) + assertEquals(id, streamedReq.id) } } diff --git a/src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/action/RequestTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/action/RequestTests.kt index 440aca087..dbf7016ea 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/action/RequestTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/action/RequestTests.kt @@ -82,22 +82,22 @@ class RequestTests : OpenSearchTestCase() { fun `test start sm policy request`() { val id = "some_id" - val req = StartSMRequest(id).index(INDEX_MANAGEMENT_INDEX) + val req = StartSMRequest(id) val out = BytesStreamOutput().apply { req.writeTo(this) } val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) val streamedReq = StartSMRequest(sin) - assertEquals(id, streamedReq.id()) + assertEquals(id, streamedReq.id) } fun `test stop sm policy request`() { val id = "some_id" - val req = StopSMRequest(id).index(INDEX_MANAGEMENT_INDEX) + val req = StopSMRequest(id) val out = BytesStreamOutput().apply { req.writeTo(this) } val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) val streamedReq = StopSMRequest(sin) - assertEquals(id, streamedReq.id()) + assertEquals(id, streamedReq.id) } fun `test explain sm policy request`() { diff --git a/src/test/kotlin/org/opensearch/indexmanagement/transform/action/RequestTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/transform/action/RequestTests.kt index e2166ccf1..79456e54d 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/transform/action/RequestTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/transform/action/RequestTests.kt @@ -151,21 +151,21 @@ class RequestTests : OpenSearchTestCase() { fun `test start transform request`() { val id = "some_id" - val req = StartTransformRequest(id).index(INDEX_MANAGEMENT_INDEX) + val req = StartTransformRequest(id) val out = BytesStreamOutput().apply { req.writeTo(this) } val streamedReq = StartTransformRequest(buildStreamInputForTransforms(out)) - assertEquals(id, streamedReq.id()) + assertEquals(id, streamedReq.id) } fun `test stop transform request`() { val id = "some_id" - val req = StopTransformRequest(id).index(INDEX_MANAGEMENT_INDEX) + val req = StopTransformRequest(id) val out = BytesStreamOutput().apply { req.writeTo(this) } val streamedReq = StopTransformRequest(buildStreamInputForTransforms(out)) - assertEquals(id, streamedReq.id()) + assertEquals(id, streamedReq.id) } } From 829d970e70fee541f604c94bc583fea18f9ab26b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 26 Dec 2024 15:30:10 -0800 Subject: [PATCH 31/37] dependabot: bump com.netflix.nebula.ospackage from 11.5.0 to 11.10.1 (#1326) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 257c51520..ac7944155 100644 --- a/build.gradle +++ b/build.gradle @@ -85,7 +85,7 @@ buildscript { } plugins { - id "com.netflix.nebula.ospackage" version "11.5.0" + id "com.netflix.nebula.ospackage" version "11.10.1" id "com.dorongold.task-tree" version "2.1.1" } From 14d4810b8e077023e98f34ce4deeb0d8983c5fd2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Dec 2024 09:17:44 -0800 Subject: [PATCH 32/37] dependabot: bump com.github.seancfoley:ipaddress from 5.4.1 to 5.5.1 (#1331) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index ac7944155..62272b53d 100644 --- a/build.gradle +++ b/build.gradle @@ -209,7 +209,7 @@ dependencies { implementation "org.jetbrains:annotations:13.0" implementation project(path: ":${rootProject.name}-spi", configuration: 'shadow') implementation "org.opensearch:common-utils:${common_utils_version}" - implementation "com.github.seancfoley:ipaddress:5.4.1" + implementation "com.github.seancfoley:ipaddress:5.5.1" implementation "commons-codec:commons-codec:${versions.commonscodec}" implementation "org.apache.httpcomponents:httpclient:${versions.httpclient}" implementation "org.apache.httpcomponents:httpcore:${versions.httpcore}" From 6bdf7cb4927066b8f776501d27191674c13e3fc2 Mon Sep 17 00:00:00 2001 From: Vikas Bansal <43470111+vikasvb90@users.noreply.github.com> Date: Thu, 2 Jan 2025 23:57:32 +0530 Subject: [PATCH 33/37] Adding new maintainers to ISM (#1321) Signed-off-by: vikasvb90 --- .github/CODEOWNERS | 2 +- MAINTAINERS.md | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index cd9176ea7..8926d78d5 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -* @bowenlan-amzn @Hailong-am @vikasvb90 +* @bowenlan-amzn @Hailong-am @vikasvb90 @tandonks @shiv0408 @soosinha diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 8127dcfa5..c76c3ea5f 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -9,6 +9,9 @@ This document contains a list of maintainers in this repo. See [opensearch-proje | Vikas Bansal | [vikasvb90](https://github.com/vikasvb90) | Amazon | | Bowen Lan | [bowenlan-amzn](https://github.com/bowenlan-amzn) | Amazon | | Hailong Cui | [Hailong-am](https://github.com/Hailong-am) | Amazon | +| Kshitij Tandon | [tandonks](https://github.com/tandonks) | Amazon | +| Shivansh Arora | [shiv0408](https://github.com/shiv0408) | Amazon | +| Sooraj Sinha | [soosinha](https://github.com/soosinha) | Amazon | ## Emeritus From 581f6dbec2602bc2771dec736e98269e2a778018 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jan 2025 09:52:41 -0800 Subject: [PATCH 34/37] dependabot: bump io.gitlab.arturbosch.detekt:detekt-gradle-plugin from 1.23.6 to 1.23.7 (#1332) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 62272b53d..d3de72b95 100644 --- a/build.gradle +++ b/build.gradle @@ -79,7 +79,7 @@ buildscript { classpath "org.opensearch.gradle:build-tools:${opensearch_version}" classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:${kotlin_version}" classpath "org.jetbrains.kotlin:kotlin-allopen:${kotlin_version}" - classpath "io.gitlab.arturbosch.detekt:detekt-gradle-plugin:1.23.6" + classpath "io.gitlab.arturbosch.detekt:detekt-gradle-plugin:1.23.7" classpath "org.jacoco:org.jacoco.agent:0.8.10" } } From 3db0eafeac0adef14007d1dfa86c7c046b925d21 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Mon, 6 Jan 2025 18:54:25 -0500 Subject: [PATCH 35/37] Get certs from security plugin instead of having them locally checked in (#1303) Co-authored-by: bowenlan-amzn --- build.gradle | 23 ++++++++++---- .../indexmanagement/ODFERestTestCase.kt | 2 +- src/test/resources/security/esnode-key.pem | 28 ------------------ src/test/resources/security/esnode.pem | 25 ---------------- src/test/resources/security/kirk-key.pem | 28 ------------------ src/test/resources/security/kirk.pem | 27 ----------------- src/test/resources/security/root-ca.pem | 28 ------------------ src/test/resources/security/sample.pem | 25 ---------------- src/test/resources/security/test-kirk.jks | Bin 3766 -> 0 bytes 9 files changed, 19 insertions(+), 167 deletions(-) delete mode 100644 src/test/resources/security/esnode-key.pem delete mode 100644 src/test/resources/security/esnode.pem delete mode 100644 src/test/resources/security/kirk-key.pem delete mode 100644 src/test/resources/security/kirk.pem delete mode 100644 src/test/resources/security/root-ca.pem delete mode 100644 src/test/resources/security/sample.pem delete mode 100644 src/test/resources/security/test-kirk.jks diff --git a/build.gradle b/build.gradle index d3de72b95..7335b8f77 100644 --- a/build.gradle +++ b/build.gradle @@ -85,6 +85,7 @@ buildscript { } plugins { + id "de.undercouch.download" version "5.3.0" id "com.netflix.nebula.ospackage" version "11.10.1" id "com.dorongold.task-tree" version "2.1.1" } @@ -189,6 +190,18 @@ tasks.named('forbiddenApisTest').configure { ext { projectSubstitutions = [:] + + ['esnode.pem', 'esnode-key.pem', 'kirk.pem', 'kirk-key.pem', 'root-ca.pem', 'sample.pem', 'test-kirk.jks'].forEach { file -> + File local = getLayout().getBuildDirectory().file(file).get().getAsFile() + download.run { + src "https://raw.githubusercontent.com/opensearch-project/security/refs/heads/main/bwc-test/src/test/resources/security/" + file + dest local + overwrite false + } + processResources { + from(local) + } + } } allprojects { @@ -342,11 +355,11 @@ afterEvaluate { plugins.add(firstPlugin) if (securityEnabled) { - node.extraConfigFile("kirk.pem", file("src/test/resources/security/kirk.pem")) - node.extraConfigFile("kirk-key.pem", file("src/test/resources/security/kirk-key.pem")) - node.extraConfigFile("esnode.pem", file("src/test/resources/security/esnode.pem")) - node.extraConfigFile("esnode-key.pem", file("src/test/resources/security/esnode-key.pem")) - node.extraConfigFile("root-ca.pem", file("src/test/resources/security/root-ca.pem")) + node.extraConfigFile("kirk.pem", file("build/resources/main/kirk.pem")) + node.extraConfigFile("kirk-key.pem", file("build/resources/main/kirk-key.pem")) + node.extraConfigFile("esnode.pem", file("build/resources/main/esnode.pem")) + node.extraConfigFile("esnode-key.pem", file("build/resources/main/esnode-key.pem")) + node.extraConfigFile("root-ca.pem", file("build/resources/main/root-ca.pem")) node.setting("plugins.security.ssl.transport.pemcert_filepath", "esnode.pem") node.setting("plugins.security.ssl.transport.pemkey_filepath", "esnode-key.pem") node.setting("plugins.security.ssl.transport.pemtrustedcas_filepath", "root-ca.pem") diff --git a/src/test/kotlin/org/opensearch/indexmanagement/ODFERestTestCase.kt b/src/test/kotlin/org/opensearch/indexmanagement/ODFERestTestCase.kt index 480e0950c..129a98f86 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/ODFERestTestCase.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/ODFERestTestCase.kt @@ -55,7 +55,7 @@ abstract class ODFERestTestCase : OpenSearchRestTestCase() { return when (keystore != null) { true -> { // create adminDN (super-admin) client - val uri = javaClass.classLoader.getResource("security/sample.pem")?.toURI() + val uri = javaClass.classLoader.getResource("sample.pem")?.toURI() val configPath = PathUtils.get(uri).parent.toAbsolutePath() SecureRestClientBuilder(settings, configPath, hosts).setSocketTimeout(60000) .setConnectionRequestTimeout(180000).build() diff --git a/src/test/resources/security/esnode-key.pem b/src/test/resources/security/esnode-key.pem deleted file mode 100644 index 567f85c93..000000000 --- a/src/test/resources/security/esnode-key.pem +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCm93kXteDQHMAv -bUPNPW5pyRHKDD42XGWSgq0k1D29C/UdyL21HLzTJa49ZU2ldIkSKs9JqbkHdyK0 -o8MO6L8dotLoYbxDWbJFW8bp1w6tDTU0HGkn47XVu3EwbfrTENg3jFu+Oem6a/50 -1SzITzJWtS0cn2dIFOBimTVpT/4Zv5qrXA6Cp4biOmoTYWhi/qQl8d0IaADiqoZ1 -MvZbZ6x76qTrRAbg+UWkpTEXoH1xTc8ndibR7+HP6OTqCKvo1NhE8uP4pY+fWd6b -6l+KLo3IKpfTbAIJXIO+M67FLtWKtttDao94B069skzKk6FPgW/OZh6PRCD0oxOa -vV+ld2SjAgMBAAECggEAQK1+uAOZeaSZggW2jQut+MaN4JHLi61RH2cFgU3COLgo -FIiNjFn8f2KKU3gpkt1It8PjlmprpYut4wHI7r6UQfuv7ZrmncRiPWHm9PB82+ZQ -5MXYqj4YUxoQJ62Cyz4sM6BobZDrjG6HHGTzuwiKvHHkbsEE9jQ4E5m7yfbVvM0O -zvwrSOM1tkZihKSTpR0j2+taji914tjBssbn12TMZQL5ItGnhR3luY8mEwT9MNkZ -xg0VcREoAH+pu9FE0vPUgLVzhJ3be7qZTTSRqv08bmW+y1plu80GbppePcgYhEow -dlW4l6XPJaHVSn1lSFHE6QAx6sqiAnBz0NoTPIaLyQKBgQDZqDOlhCRciMRicSXn -7yid9rhEmdMkySJHTVFOidFWwlBcp0fGxxn8UNSBcXdSy7GLlUtH41W9PWl8tp9U -hQiiXORxOJ7ZcB80uNKXF01hpPj2DpFPWyHFxpDkWiTAYpZl68rOlYujxZUjJIej -VvcykBC2BlEOG9uZv2kxcqLyJwKBgQDEYULTxaTuLIa17wU3nAhaainKB3vHxw9B -Ksy5p3ND43UNEKkQm7K/WENx0q47TA1mKD9i+BhaLod98mu0YZ+BCUNgWKcBHK8c -uXpauvM/pLhFLXZ2jvEJVpFY3J79FSRK8bwE9RgKfVKMMgEk4zOyZowS8WScOqiy -hnQn1vKTJQKBgElhYuAnl9a2qXcC7KOwRsJS3rcKIVxijzL4xzOyVShp5IwIPbOv -hnxBiBOH/JGmaNpFYBcBdvORE9JfA4KMQ2fx53agfzWRjoPI1/7mdUk5RFI4gRb/ -A3jZRBoopgFSe6ArCbnyQxzYzToG48/Wzwp19ZxYrtUR4UyJct6f5n27AoGBAJDh -KIpQQDOvCdtjcbfrF4aM2DPCfaGPzENJriwxy6oEPzDaX8Bu/dqI5Ykt43i/zQrX -GpyLaHvv4+oZVTiI5UIvcVO9U8hQPyiz9f7F+fu0LHZs6f7hyhYXlbe3XFxeop3f -5dTKdWgXuTTRF2L9dABkA2deS9mutRKwezWBMQk5AoGBALPtX0FrT1zIosibmlud -tu49A/0KZu4PBjrFMYTSEWGNJez3Fb2VsJwylVl6HivwbP61FhlYfyksCzQQFU71 -+x7Nmybp7PmpEBECr3deoZKQ/acNHn0iwb0It+YqV5+TquQebqgwK6WCLsMuiYKT -bg/ch9Rhxbq22yrVgWHh6epp ------END PRIVATE KEY----- diff --git a/src/test/resources/security/esnode.pem b/src/test/resources/security/esnode.pem deleted file mode 100644 index b690a603d..000000000 --- a/src/test/resources/security/esnode.pem +++ /dev/null @@ -1,25 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIEPDCCAySgAwIBAgIUaYSlET3nzsotWTrWueVPPh10yLYwDQYJKoZIhvcNAQEL -BQAwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJk/IsZAEZFgdleGFt -cGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYDVQQLDBhFeGFtcGxl -IENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUgQ29tIEluYy4gUm9v -dCBDQTAeFw0yNDAyMjAxNzAzMjVaFw0zNDAyMTcxNzAzMjVaMFcxCzAJBgNVBAYT -AmRlMQ0wCwYDVQQHDAR0ZXN0MQ0wCwYDVQQKDARub2RlMQ0wCwYDVQQLDARub2Rl -MRswGQYDVQQDDBJub2RlLTAuZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA -A4IBDwAwggEKAoIBAQCm93kXteDQHMAvbUPNPW5pyRHKDD42XGWSgq0k1D29C/Ud -yL21HLzTJa49ZU2ldIkSKs9JqbkHdyK0o8MO6L8dotLoYbxDWbJFW8bp1w6tDTU0 -HGkn47XVu3EwbfrTENg3jFu+Oem6a/501SzITzJWtS0cn2dIFOBimTVpT/4Zv5qr -XA6Cp4biOmoTYWhi/qQl8d0IaADiqoZ1MvZbZ6x76qTrRAbg+UWkpTEXoH1xTc8n -dibR7+HP6OTqCKvo1NhE8uP4pY+fWd6b6l+KLo3IKpfTbAIJXIO+M67FLtWKtttD -ao94B069skzKk6FPgW/OZh6PRCD0oxOavV+ld2SjAgMBAAGjgcYwgcMwRwYDVR0R -BEAwPogFKgMEBQWCEm5vZGUtMC5leGFtcGxlLmNvbYIJbG9jYWxob3N0hxAAAAAA -AAAAAAAAAAAAAAABhwR/AAABMAsGA1UdDwQEAwIF4DAdBgNVHSUEFjAUBggrBgEF -BQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQU0/qDQaY10jIo -wCjLUpz/HfQXyt8wHwYDVR0jBBgwFoAUF4ffoFrrZhKn1dD4uhJFPLcrAJwwDQYJ -KoZIhvcNAQELBQADggEBAGbij5WyF0dKhQodQfTiFDb73ygU6IyeJkFSnxF67gDz -pQJZKFvXuVBa3cGP5e7Qp3TK50N+blXGH0xXeIV9lXeYUk4hVfBlp9LclZGX8tGi -7Xa2enMvIt5q/Yg3Hh755ZxnDYxCoGkNOXUmnMusKstE0YzvZ5Gv6fcRKFBUgZLh -hUBqIEAYly1EqH/y45APiRt3Nor1yF6zEI4TnL0yNrHw6LyQkUNCHIGMJLfnJQ9L -camMGIXOx60kXNMTigF9oXXwixWAnDM9y3QT8QXA7hej/4zkbO+vIeV/7lGUdkyg -PAi92EvyxmsliEMyMR0VINl8emyobvfwa7oMeWMR+hg= ------END CERTIFICATE----- diff --git a/src/test/resources/security/kirk-key.pem b/src/test/resources/security/kirk-key.pem deleted file mode 100644 index fd1728cda..000000000 --- a/src/test/resources/security/kirk-key.pem +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCVXDgEJQorgfXp -gpY0TgF55bD2xuzxN5Dc9rDfgWxrsOvOloMpd7k6FR71bKWjJi1KptSmM/cDElky -AWYKSfYWGiGxsQ+EQW+6kwCfEOHXQldn+0+JcWqP+osSPjtJfwRvRN5kRqP69MPo -7U0N2kdqenqMWjmG1chDGLRSOEGU5HIBiDxsZtOcvMaJ8b1eaW0lvS+6gFQ80AvB -GBkDDCOHHLtDXBylrZk2CQP8AzxNicIZ4B8G3CG3OHA8+nBtEtxZoIihrrkqlMt+ -b/5N8u8zB0Encew0kdrc4R/2wS//ahr6U+9Siq8T7WsUtGwKj3BJClg6OyDJRhlu -y2gFnxoPAgMBAAECggEAP5TOycDkx+megAWVoHV2fmgvgZXkBrlzQwUG/VZQi7V4 -ZGzBMBVltdqI38wc5MtbK3TCgHANnnKgor9iq02Z4wXDwytPIiti/ycV9CDRKvv0 -TnD2hllQFjN/IUh5n4thHWbRTxmdM7cfcNgX3aZGkYbLBVVhOMtn4VwyYu/Mxy8j -xClZT2xKOHkxqwmWPmdDTbAeZIbSv7RkIGfrKuQyUGUaWhrPslvYzFkYZ0umaDgQ -OAthZew5Bz3OfUGOMPLH61SVPuJZh9zN1hTWOvT65WFWfsPd2yStI+WD/5PU1Doo -1RyeHJO7s3ug8JPbtNJmaJwHe9nXBb/HXFdqb976yQKBgQDNYhpu+MYSYupaYqjs -9YFmHQNKpNZqgZ4ceRFZ6cMJoqpI5dpEMqToFH7tpor72Lturct2U9nc2WR0HeEs -/6tiptyMPTFEiMFb1opQlXF2ae7LeJllntDGN0Q6vxKnQV+7VMcXA0Y8F7tvGDy3 -qJu5lfvB1mNM2I6y/eMxjBuQhwKBgQC6K41DXMFro0UnoO879pOQYMydCErJRmjG -/tZSy3Wj4KA/QJsDSViwGfvdPuHZRaG9WtxdL6kn0w1exM9Rb0bBKl36lvi7o7xv -M+Lw9eyXMkww8/F5d7YYH77gIhGo+RITkKI3+5BxeBaUnrGvmHrpmpgRXWmINqr0 -0jsnN3u0OQKBgCf45vIgItSjQb8zonLz2SpZjTFy4XQ7I92gxnq8X0Q5z3B+o7tQ -K/4rNwTju/sGFHyXAJlX+nfcK4vZ4OBUJjP+C8CTjEotX4yTNbo3S6zjMyGQqDI5 -9aIOUY4pb+TzeUFJX7If5gR+DfGyQubvvtcg1K3GHu9u2l8FwLj87sRzAoGAflQF -RHuRiG+/AngTPnZAhc0Zq0kwLkpH2Rid6IrFZhGLy8AUL/O6aa0IGoaMDLpSWUJp -nBY2S57MSM11/MVslrEgGmYNnI4r1K25xlaqV6K6ztEJv6n69327MS4NG8L/gCU5 -3pEm38hkUi8pVYU7in7rx4TCkrq94OkzWJYurAkCgYATQCL/rJLQAlJIGulp8s6h -mQGwy8vIqMjAdHGLrCS35sVYBXG13knS52LJHvbVee39AbD5/LlWvjJGlQMzCLrw -F7oILW5kXxhb8S73GWcuMbuQMFVHFONbZAZgn+C9FW4l7XyRdkrbR1MRZ2km8YMs -/AHmo368d4PSNRMMzLHw8Q== ------END PRIVATE KEY----- diff --git a/src/test/resources/security/kirk.pem b/src/test/resources/security/kirk.pem deleted file mode 100644 index b89edfe18..000000000 --- a/src/test/resources/security/kirk.pem +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIEmDCCA4CgAwIBAgIUaYSlET3nzsotWTrWueVPPh10yLcwDQYJKoZIhvcNAQEL -BQAwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJk/IsZAEZFgdleGFt -cGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYDVQQLDBhFeGFtcGxl -IENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUgQ29tIEluYy4gUm9v -dCBDQTAeFw0yNDAyMjAxNzA0MjRaFw0zNDAyMTcxNzA0MjRaME0xCzAJBgNVBAYT -AmRlMQ0wCwYDVQQHDAR0ZXN0MQ8wDQYDVQQKDAZjbGllbnQxDzANBgNVBAsMBmNs -aWVudDENMAsGA1UEAwwEa2lyazCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC -ggEBAJVcOAQlCiuB9emCljROAXnlsPbG7PE3kNz2sN+BbGuw686Wgyl3uToVHvVs -paMmLUqm1KYz9wMSWTIBZgpJ9hYaIbGxD4RBb7qTAJ8Q4ddCV2f7T4lxao/6ixI+ -O0l/BG9E3mRGo/r0w+jtTQ3aR2p6eoxaOYbVyEMYtFI4QZTkcgGIPGxm05y8xonx -vV5pbSW9L7qAVDzQC8EYGQMMI4ccu0NcHKWtmTYJA/wDPE2JwhngHwbcIbc4cDz6 -cG0S3FmgiKGuuSqUy35v/k3y7zMHQSdx7DSR2tzhH/bBL/9qGvpT71KKrxPtaxS0 -bAqPcEkKWDo7IMlGGW7LaAWfGg8CAwEAAaOCASswggEnMAwGA1UdEwEB/wQCMAAw -DgYDVR0PAQH/BAQDAgXgMBYGA1UdJQEB/wQMMAoGCCsGAQUFBwMCMB0GA1UdDgQW -BBSjMS8tgguX/V7KSGLoGg7K6XMzIDCBzwYDVR0jBIHHMIHEgBQXh9+gWutmEqfV -0Pi6EkU8tysAnKGBlaSBkjCBjzETMBEGCgmSJomT8ixkARkWA2NvbTEXMBUGCgmS -JomT8ixkARkWB2V4YW1wbGUxGTAXBgNVBAoMEEV4YW1wbGUgQ29tIEluYy4xITAf -BgNVBAsMGEV4YW1wbGUgQ29tIEluYy4gUm9vdCBDQTEhMB8GA1UEAwwYRXhhbXBs -ZSBDb20gSW5jLiBSb290IENBghQNZAmZZn3EFOxBR4630XlhI+mo4jANBgkqhkiG -9w0BAQsFAAOCAQEACEUPPE66/Ot3vZqRGpjDjPHAdtOq+ebaglQhvYcnDw8LOZm8 -Gbh9M88CiO6UxC8ipQLTPh2yyeWArkpJzJK/Pi1eoF1XLiAa0sQ/RaJfQWPm9dvl -1ZQeK5vfD4147b3iBobwEV+CR04SKow0YeEEzAJvzr8YdKI6jqr+2GjjVqzxvRBy -KRVHWCFiR7bZhHGLq3br8hSu0hwjb3oGa1ZI8dui6ujyZt6nm6BoEkau3G/6+zq9 -E6vX3+8Fj4HKCAL6i0SwfGmEpTNp5WUhqibK/fMhhmMT4Mx6MxkT+OFnIjdUU0S/ -e3kgnG8qjficUr38CyEli1U0M7koIXUZI7r+LQ== ------END CERTIFICATE----- diff --git a/src/test/resources/security/root-ca.pem b/src/test/resources/security/root-ca.pem deleted file mode 100644 index 854323e6f..000000000 --- a/src/test/resources/security/root-ca.pem +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIExjCCA66gAwIBAgIUDWQJmWZ9xBTsQUeOt9F5YSPpqOIwDQYJKoZIhvcNAQEL -BQAwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJk/IsZAEZFgdleGFt -cGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYDVQQLDBhFeGFtcGxl -IENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUgQ29tIEluYy4gUm9v -dCBDQTAeFw0yNDAyMjAxNzAwMzZaFw0zNDAyMTcxNzAwMzZaMIGPMRMwEQYKCZIm -iZPyLGQBGRYDY29tMRcwFQYKCZImiZPyLGQBGRYHZXhhbXBsZTEZMBcGA1UECgwQ -RXhhbXBsZSBDb20gSW5jLjEhMB8GA1UECwwYRXhhbXBsZSBDb20gSW5jLiBSb290 -IENBMSEwHwYDVQQDDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0EwggEiMA0GCSqG -SIb3DQEBAQUAA4IBDwAwggEKAoIBAQDEPyN7J9VGPyJcQmCBl5TGwfSzvVdWwoQU -j9aEsdfFJ6pBCDQSsj8Lv4RqL0dZra7h7SpZLLX/YZcnjikrYC+rP5OwsI9xEE/4 -U98CsTBPhIMgqFK6SzNE5494BsAk4cL72dOOc8tX19oDS/PvBULbNkthQ0aAF1dg -vbrHvu7hq7LisB5ZRGHVE1k/AbCs2PaaKkn2jCw/b+U0Ml9qPuuEgz2mAqJDGYoA -WSR4YXrOcrmPuRqbws464YZbJW898/0Pn/U300ed+4YHiNYLLJp51AMkR4YEw969 -VRPbWIvLrd0PQBooC/eLrL6rvud/GpYhdQEUx8qcNCKd4bz3OaQ5AgMBAAGjggEW -MIIBEjAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjAdBgNVHQ4EFgQU -F4ffoFrrZhKn1dD4uhJFPLcrAJwwgc8GA1UdIwSBxzCBxIAUF4ffoFrrZhKn1dD4 -uhJFPLcrAJyhgZWkgZIwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJ -k/IsZAEZFgdleGFtcGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYD -VQQLDBhFeGFtcGxlIENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUg -Q29tIEluYy4gUm9vdCBDQYIUDWQJmWZ9xBTsQUeOt9F5YSPpqOIwDQYJKoZIhvcN -AQELBQADggEBAL3Q3AHUhMiLUy6OlLSt8wX9I2oNGDKbBu0atpUNDztk/0s3YLQC -YuXgN4KrIcMXQIuAXCx407c+pIlT/T1FNn+VQXwi56PYzxQKtlpoKUL3oPQE1d0V -6EoiNk+6UodvyZqpdQu7fXVentRMk1QX7D9otmiiNuX+GSxJhJC2Lyzw65O9EUgG -1yVJon6RkUGtqBqKIuLksKwEr//ELnjmXit4LQKSnqKr0FTCB7seIrKJNyb35Qnq -qy9a/Unhokrmdda1tr6MbqU8l7HmxLuSd/Ky+L0eDNtYv6YfMewtjg0TtAnFyQov -rdXmeq1dy9HLo3Ds4AFz3Gx9076TxcRS/iI= ------END CERTIFICATE----- diff --git a/src/test/resources/security/sample.pem b/src/test/resources/security/sample.pem deleted file mode 100644 index b690a603d..000000000 --- a/src/test/resources/security/sample.pem +++ /dev/null @@ -1,25 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIEPDCCAySgAwIBAgIUaYSlET3nzsotWTrWueVPPh10yLYwDQYJKoZIhvcNAQEL -BQAwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJk/IsZAEZFgdleGFt -cGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYDVQQLDBhFeGFtcGxl -IENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUgQ29tIEluYy4gUm9v -dCBDQTAeFw0yNDAyMjAxNzAzMjVaFw0zNDAyMTcxNzAzMjVaMFcxCzAJBgNVBAYT -AmRlMQ0wCwYDVQQHDAR0ZXN0MQ0wCwYDVQQKDARub2RlMQ0wCwYDVQQLDARub2Rl -MRswGQYDVQQDDBJub2RlLTAuZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA -A4IBDwAwggEKAoIBAQCm93kXteDQHMAvbUPNPW5pyRHKDD42XGWSgq0k1D29C/Ud -yL21HLzTJa49ZU2ldIkSKs9JqbkHdyK0o8MO6L8dotLoYbxDWbJFW8bp1w6tDTU0 -HGkn47XVu3EwbfrTENg3jFu+Oem6a/501SzITzJWtS0cn2dIFOBimTVpT/4Zv5qr -XA6Cp4biOmoTYWhi/qQl8d0IaADiqoZ1MvZbZ6x76qTrRAbg+UWkpTEXoH1xTc8n -dibR7+HP6OTqCKvo1NhE8uP4pY+fWd6b6l+KLo3IKpfTbAIJXIO+M67FLtWKtttD -ao94B069skzKk6FPgW/OZh6PRCD0oxOavV+ld2SjAgMBAAGjgcYwgcMwRwYDVR0R -BEAwPogFKgMEBQWCEm5vZGUtMC5leGFtcGxlLmNvbYIJbG9jYWxob3N0hxAAAAAA -AAAAAAAAAAAAAAABhwR/AAABMAsGA1UdDwQEAwIF4DAdBgNVHSUEFjAUBggrBgEF -BQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQU0/qDQaY10jIo -wCjLUpz/HfQXyt8wHwYDVR0jBBgwFoAUF4ffoFrrZhKn1dD4uhJFPLcrAJwwDQYJ -KoZIhvcNAQELBQADggEBAGbij5WyF0dKhQodQfTiFDb73ygU6IyeJkFSnxF67gDz -pQJZKFvXuVBa3cGP5e7Qp3TK50N+blXGH0xXeIV9lXeYUk4hVfBlp9LclZGX8tGi -7Xa2enMvIt5q/Yg3Hh755ZxnDYxCoGkNOXUmnMusKstE0YzvZ5Gv6fcRKFBUgZLh -hUBqIEAYly1EqH/y45APiRt3Nor1yF6zEI4TnL0yNrHw6LyQkUNCHIGMJLfnJQ9L -camMGIXOx60kXNMTigF9oXXwixWAnDM9y3QT8QXA7hej/4zkbO+vIeV/7lGUdkyg -PAi92EvyxmsliEMyMR0VINl8emyobvfwa7oMeWMR+hg= ------END CERTIFICATE----- diff --git a/src/test/resources/security/test-kirk.jks b/src/test/resources/security/test-kirk.jks deleted file mode 100644 index 6c8c5ef77e20980f8c78295b159256b805da6a28..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3766 zcmd^=c{r47AIImJ%`(PV###wuU&o%k$xbMgr4m`Pk2Tv-j4?=zEwY?!X|aVw)I`=A zPAY52Rt6yODkPjhAQ%WsfbL*f;mp!-018Nf*#Q6sf)b!}Nv;s_8gzOC@mTmi+D9F}jyYkhL=#Xk3eYM2csmxKA&W!xAdE{tZ2mEGS z;L%QU`DHcrbdbw$3GsKUvmfQu0Z^?sH7B)!W)eLbG*fXB^G$&6CbCnj4~ z*J>Rkut6vL1EvT!JqAq#X=O~#!JHQ#QVSPuOGlnLrXXB~{{FsGRq?o?I;>^GFEhMB zw;z!v1sXap8nq3zz&+prKs-DRPm*XsS4BaP6Z{8tM~n@m|rxMA=p6*i(w=7 z*2&*Yg-uWU$5|W>>g5h)Fn{3B={`skAJ5_wXB5pDwyj{vG1_{{Y-`wB_i^B!5PA|= zrx=_>rprb&75BQ=J)SKPAJI;?(D#46)o+a?SsR^-&qJjXY2ER8S*1ZvU`t7~M6?NKULuzlAZ8C#X9>8j2;WDY z(TY-^!`&0%67`u|U_-Y(knWVcSlh-kwZQ6KG@S?L`W!iVl>Gyd(LnpMc@C!QeY{(E z)uAwF_CcqH#00}jer2dQk3}R|p^87XCxR8`n4c@g9rASTt9$8}SuGW!!+QQ&w&G!P zvv5Mft<&pzv^&XuuQAj&ieoa*3nI-hx}0`4kym=(cd>?v6yM3v43y@5@;yPeJ_N{@ z622W$@5Z4VqliMF3GAf_RcB;$HX^%cwTCgxg^4)5I0?*&oW|giBB@nUNBO+IX=iON zo~;L}HOwhyeqH4GHvAQ5i=|0c+_5*661aDyT_tr=I#+Zog%!9nRiuBb8m&SS4qp2fv7HJMG zwJFuqV*Hoq3`|Mayml;So|9W4Um6Lu8(k+(Hc2}p@&>?!7!7H~9*O%@BrKNAOa-~e z$e6#G)fJ+wNz5x9zU;#>&V}d z?!F1W_eNN;&LI9$!kWa0Zqa)0CVM4D=x(r>aXgW=XQ)PTRsJJ&MC?WjjoMwLRh`-I z8yD|^&(r#NU|pRpRF%wn&t%X`)8HQe%uxEKnXxIu9yui1s$eH0*YZ^Wvt25yOg6{5 zPefKstjqam-PRDz=&-BVb^xZe>{C{$cza!_sV&3M*l0ocMJVr!l~TlJi4JChDn9Nn zc&la1caY}0P&Ho=r;)l;mKBf$V<6A*R6XC}s98g%I7ZIAFI=e6SqQ4;oevw)nw0%^ zKq9#$;{3R0zJv}#mr7@}e+5-(`{C?^vEE#xb7uBY=X#_1v+@~@l?W@Zaq+Yo9bpu& zR<0us_T`(Q6qp1xYb)Rq;tJ|aTZ&y5xqx<_j-|>1$SEi@3!A|| z9YH<3ub_#ai=2WG_V9iQ!NU8mB|$4ZK3Gr>_s15;6W-XV-*##3TjwoMP&yb zq!L{!sQoUn<_ZWb)BbzloM2Zs1tb=+FBn*$!EQmp3Ml#oe;g0);^XP&_osni`NR1A z0SL>FG{F)8;h%d#4-g0eK+%&0UD-=ghUr~yDQ?!lNE5tKiJ_rjY{@`Q1vjbVAFU;|?Qs;w|1hFx_ z`*jR7rVAU>9*yRSpD1)#aOb!)@ak(5hk;guG$_9)=K8Ie^uOP<63|FjrX2UEcJw07 zD5c?bxHD${?)1+CMgPg@0|kH>4NzJZO*;#rl-xA_8*SHCS}ygKZP7*uHbRtmaTE%n zp7Vt7QIt|IIN?)fyS#8IxKHO$?TeY{DpQl5^kyAd$HH^Aa)SJC+I0!ULR znF7*z6R6~{CCW6M^qKuU!N`I`>YB3i6toA7f7#3%T&$5&wm0nY{&d9(g)LB$%g9dX zf>HfjVn9;)rG-^=)tiGDd<5M4wDHPl@yEGU_whSh78l$%S*WCqjvj^Xt?_VKp0T{pQGU!F;?_^4EMT$__$E zH0hMGQlo@W2p^_tPZsnirl@pGb<#0a^*g5ihYtSzKKx%Wg;i4h8B_c6Z+PPWM!I%g zOr-dLp|0@RV@@&InVrwRJfPT~ZY840gT$Jl4)HP^qcTUWE~1&}C2wS3Sv9pJWiRva zyK}a9ilnrYe7SB$bu~GF&GM`D1h@ukNsJY|Yt>|?q(4gzgSUuGwSIfsmlD)%J2V0@ zTU&-58&x%P)-#Oev2~&}bv^wwRbD$?Enu(jJiuwM3shGOZ{$juY+RGk#m^`!p7+vO zAjWFn1{dq`T?N^TggHmN3~VGf^5?a_)R-cj5yfk-?V<|S)%uKn{YGL)7(~eAhWA56 zj7ZS7amp#qQM;t>%6F)v{1S-Gq>88IPiL?2X9=q_r$vhc4{Pd3$WssBMbZaV2W zu&8||{U99-3!x+JudoA1KSAx^0qg$*YLr)FKtJ($lC@k)W?khPY!~B&3F~Xnxs_WH)b*(MC{~@>r={U4@A6+2p8il>0lojdT`r8~C>rA6;jw^lZK9gk<_y!v za(Rbclc{1;TFBtT`lr|YO0}|UXzh>FLsx6RQUq8=?V4{NR#=oxL2}kHb-ZAfuN Date: Mon, 13 Jan 2025 17:33:11 +0530 Subject: [PATCH 36/37] dependabot: bump commons-codec:commons-codec from 1.13 to 1.17.2 (#1334) Bumps [commons-codec:commons-codec](https://github.com/apache/commons-codec) from 1.13 to 1.17.2. - [Changelog](https://github.com/apache/commons-codec/blob/master/RELEASE-NOTES.txt) - [Commits](https://github.com/apache/commons-codec/compare/commons-codec-1.13...rel/commons-codec-1.17.2) --- updated-dependencies: - dependency-name: commons-codec:commons-codec dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 7335b8f77..f678928ea 100644 --- a/build.gradle +++ b/build.gradle @@ -113,7 +113,7 @@ configurations.all { force 'commons-beanutils:commons-beanutils:1.9.4' force 'com.google.guava:guava:30.0-jre' force 'com.puppycrawl.tools:checkstyle:8.29' - force 'commons-codec:commons-codec:1.13' + force 'commons-codec:commons-codec:1.17.2' force "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" force "org.yaml:snakeyaml:${versions.snakeyaml}" force 'org.codehaus.plexus:plexus-utils:3.0.24' From 57eb025bc80a5e1e41a72870c385ffd7f6441c25 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Jan 2025 17:34:26 +0530 Subject: [PATCH 37/37] dependabot: bump de.undercouch.download from 5.3.0 to 5.6.0 (#1333) Bumps de.undercouch.download from 5.3.0 to 5.6.0. --- updated-dependencies: - dependency-name: de.undercouch.download dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index f678928ea..840eccb64 100644 --- a/build.gradle +++ b/build.gradle @@ -85,7 +85,7 @@ buildscript { } plugins { - id "de.undercouch.download" version "5.3.0" + id "de.undercouch.download" version "5.6.0" id "com.netflix.nebula.ospackage" version "11.10.1" id "com.dorongold.task-tree" version "2.1.1" }