From 5b16480657bb32e3a5560d63be61bf3c81c8f50a Mon Sep 17 00:00:00 2001 From: Santhosh Gandhe <1909520+san81@users.noreply.github.com> Date: Fri, 31 Jan 2025 12:15:40 -0800 Subject: [PATCH] Plugin metrics injection (#5372) * injectable plugin metrics Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com> * removed an unused parameter Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com> * fixing a flaky test Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com> --------- Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com> --- .../plugin/DefaultPluginFactoryIT.java | 15 ++++++---- .../plugin/DefaultPluginFactory.java | 8 ++--- .../plugin/PluginBeanFactoryProvider.java | 16 ++++++---- .../plugin/DefaultPluginFactoryTest.java | 26 ++++++++-------- .../plugin/PluginBeanFactoryProviderTest.java | 22 +++++++------- .../TestComponentWithConfigInject.java | 9 +++++- .../oteltrace/OTelTraceRawProcessorTest.java | 9 ++++-- .../plugins/source/jira/JiraService.java | 9 +++--- .../source/jira/rest/JiraRestClient.java | 25 ++++++++-------- .../plugins/source/jira/JiraIteratorTest.java | 20 ++++++------- .../plugins/source/jira/JiraServiceTest.java | 22 +++++++------- .../source/jira/rest/JiraRestClientTest.java | 30 ++++++++----------- .../source/source_crawler/base/Crawler.java | 4 +-- .../scheduler/WorkerScheduler.java | 5 ++-- .../source_crawler/base/CrawlerTest.java | 4 ++- 15 files changed, 118 insertions(+), 106 deletions(-) diff --git a/data-prepper-core/src/integrationTest/java/org/opensearch/dataprepper/plugin/DefaultPluginFactoryIT.java b/data-prepper-core/src/integrationTest/java/org/opensearch/dataprepper/plugin/DefaultPluginFactoryIT.java index 9230aa7ff8..de81283f0b 100644 --- a/data-prepper-core/src/integrationTest/java/org/opensearch/dataprepper/plugin/DefaultPluginFactoryIT.java +++ b/data-prepper-core/src/integrationTest/java/org/opensearch/dataprepper/plugin/DefaultPluginFactoryIT.java @@ -5,6 +5,7 @@ package org.opensearch.dataprepper.plugin; +import io.micrometer.core.instrument.Counter; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -14,18 +15,19 @@ import org.opensearch.dataprepper.core.event.EventFactoryApplicationContextMarker; import org.opensearch.dataprepper.core.validation.LoggingPluginErrorsHandler; import org.opensearch.dataprepper.core.validation.PluginErrorCollector; -import org.opensearch.dataprepper.model.plugin.NoPluginFoundException; -import org.opensearch.dataprepper.plugins.configtest.TestComponentWithConfigInject; -import org.opensearch.dataprepper.plugins.configtest.TestDISourceWithConfig; -import org.opensearch.dataprepper.validation.PluginErrorsHandler; +import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.configuration.PipelinesDataFlowModel; import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.model.plugin.InvalidPluginConfigurationException; +import org.opensearch.dataprepper.model.plugin.NoPluginFoundException; import org.opensearch.dataprepper.model.source.Source; import org.opensearch.dataprepper.plugins.TestObjectPlugin; +import org.opensearch.dataprepper.plugins.configtest.TestComponentWithConfigInject; +import org.opensearch.dataprepper.plugins.configtest.TestDISourceWithConfig; import org.opensearch.dataprepper.plugins.test.TestComponent; import org.opensearch.dataprepper.plugins.test.TestDISource; import org.opensearch.dataprepper.plugins.test.TestPlugin; +import org.opensearch.dataprepper.validation.PluginErrorsHandler; import org.springframework.context.annotation.AnnotationConfigApplicationContext; import java.util.Collections; @@ -129,7 +131,7 @@ void loadPlugin_should_return_a_new_plugin_instance_with_DI_context_initialized( } @Test - void loadPlugin_should_return_a_new_plugin_instance_with_DI_context_and_config_injected() { + void loadPlugin_should_return_a_new_plugin_instance_with_DI_context_with_config_and_plugin_metrics_injected() { final String requiredStringValue = UUID.randomUUID().toString(); final String optionalStringValue = UUID.randomUUID().toString(); @@ -152,6 +154,9 @@ void loadPlugin_should_return_a_new_plugin_instance_with_DI_context_and_config_i assertThat(pluginConfig.getRequiredString(), equalTo(requiredStringValue)); assertThat(pluginConfig.getOptionalString(), equalTo(optionalStringValue)); assertThat(plugin.getTestComponent().getIdentifier(), equalTo("test-component-with-plugin-config-injected")); + PluginMetrics pluginMetrics = plugin.getTestComponent().getPluginMetrics(); + assertInstanceOf(PluginMetrics.class, pluginMetrics); + assertInstanceOf(Counter.class, pluginMetrics.counter("testCounter")); } @Test diff --git a/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/DefaultPluginFactory.java b/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/DefaultPluginFactory.java index 0ec3b5a953..456a0ef7be 100644 --- a/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/DefaultPluginFactory.java +++ b/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/DefaultPluginFactory.java @@ -64,14 +64,14 @@ public class DefaultPluginFactory implements PluginFactory { this.pluginBeanFactoryProvider = Objects.requireNonNull(pluginBeanFactoryProvider); this.pluginConfigurationObservableFactory = pluginConfigurationObservableFactory; - if(pluginProviders.isEmpty()) { + if (pluginProviders.isEmpty()) { throw new RuntimeException("Data Prepper requires at least one PluginProvider. " + "Your Data Prepper configuration may be missing the org.opensearch.dataprepper.plugin.PluginProvider file."); } } @Override - public T loadPlugin(final Class baseClass, final PluginSetting pluginSetting, final Object ... args) { + public T loadPlugin(final Class baseClass, final PluginSetting pluginSetting, final Object... args) { final String pluginName = pluginSetting.getName(); final Class pluginClass = getPluginClass(baseClass, pluginName); @@ -100,7 +100,7 @@ public List loadPlugins( final Integer numberOfInstances = numberOfInstancesFunction.apply(pluginClass); - if(numberOfInstances == null || numberOfInstances < 0) + if (numberOfInstances == null || numberOfInstances < 0) throw new IllegalArgumentException("The numberOfInstances must be provided as a non-negative integer."); final ComponentPluginArgumentsContext constructionContext = getConstructionContext(pluginSetting, pluginClass, null); @@ -121,7 +121,7 @@ private ComponentPluginArgumentsContext getConstructionContext(final PluginS .createDefaultPluginConfigObservable(pluginConfigurationConverter, pluginConfigurationType, pluginSetting); Class[] markersToScan = pluginAnnotation.packagesToScan(); - BeanFactory beanFactory = pluginBeanFactoryProvider.createPluginSpecificContext(markersToScan, configuration); + BeanFactory beanFactory = pluginBeanFactoryProvider.createPluginSpecificContext(markersToScan, configuration, pluginSetting); return new ComponentPluginArgumentsContext.Builder() .withPluginSetting(pluginSetting) diff --git a/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/PluginBeanFactoryProvider.java b/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/PluginBeanFactoryProvider.java index 76762f8d8e..c544c99d3f 100644 --- a/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/PluginBeanFactoryProvider.java +++ b/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/PluginBeanFactoryProvider.java @@ -5,6 +5,7 @@ package org.opensearch.dataprepper.plugin; +import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.springframework.beans.factory.BeanFactory; import org.springframework.beans.factory.support.DefaultListableBeanFactory; @@ -20,8 +21,8 @@ /** * @since 1.3 *

- * Used to create new instances of ApplicationContext that can be used to provide a per plugin instance isolated ApplicationContext - * scope. CoreApplicationContext is unavailable to sharedPluginApplicationContext and its children. + * Used to create new instances of ApplicationContext that can be used to provide a per plugin instance isolated ApplicationContext + * scope. CoreApplicationContext is unavailable to sharedPluginApplicationContext and its children. *

*

pluginIsolatedApplicationContext inherits from {@link PluginBeanFactoryProvider#sharedPluginApplicationContext}

*

{@link PluginBeanFactoryProvider#sharedPluginApplicationContext} inherits from publicContext

@@ -53,20 +54,23 @@ GenericApplicationContext getCoreApplicationContext() { } /** + * @return BeanFactory A BeanFactory that inherits from {@link PluginBeanFactoryProvider#sharedPluginApplicationContext} * @since 1.3 * Creates a new isolated application context that inherits from * {@link PluginBeanFactoryProvider#sharedPluginApplicationContext} then returns new context's BeanFactory. * {@link PluginBeanFactoryProvider#sharedPluginApplicationContext} should not be directly accessible to plugins. * instead, a new isolated {@link ApplicationContext} should be created. - * @return BeanFactory A BeanFactory that inherits from {@link PluginBeanFactoryProvider#sharedPluginApplicationContext} */ - public BeanFactory createPluginSpecificContext(Class[] markersToScan, Object configuration) { + public BeanFactory createPluginSpecificContext(Class[] markersToScan, Object configuration, final PluginSetting pluginSetting) { AnnotationConfigApplicationContext isolatedPluginApplicationContext = new AnnotationConfigApplicationContext(); DefaultListableBeanFactory beanFactory = (DefaultListableBeanFactory) isolatedPluginApplicationContext.getBeanFactory(); - if(markersToScan !=null && markersToScan.length>0) { - if(configuration !=null && !(configuration instanceof PluginSetting)) { + if (markersToScan != null && markersToScan.length > 0) { + if (configuration != null && !(configuration instanceof PluginSetting)) { beanFactory.registerSingleton(configuration.getClass().getName(), configuration); } + if (pluginSetting != null) { + beanFactory.registerSingleton(PluginMetrics.class.getName(), PluginMetrics.fromPluginSetting(pluginSetting)); + } // If packages to scan is provided in this plugin annotation, which indicates // that this plugin is interested in using Dependency Injection isolated for its module Arrays.stream(markersToScan) diff --git a/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/DefaultPluginFactoryTest.java b/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/DefaultPluginFactoryTest.java index 2c1bf9e0fa..3ae13b83f4 100644 --- a/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/DefaultPluginFactoryTest.java +++ b/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/DefaultPluginFactoryTest.java @@ -216,7 +216,7 @@ void loadPlugin_should_create_a_new_instance_of_the_plugin_with_di_initialized() equalTo(expectedInstance)); verify(pluginConfigurationObservableFactory).createDefaultPluginConfigObservable(eq(pluginConfigurationConverter), eq(PluginSetting.class), eq(pluginSetting)); - verify(beanFactoryProvider).createPluginSpecificContext(new Class[]{TestDISource.class}, convertedConfiguration); + verify(beanFactoryProvider).createPluginSpecificContext(new Class[]{TestDISource.class}, convertedConfiguration, pluginSetting); } @Test @@ -233,7 +233,7 @@ void loadPlugin_should_create_a_new_instance_of_the_first_plugin_found() { equalTo(expectedInstance)); verify(pluginConfigurationObservableFactory).createDefaultPluginConfigObservable(eq(pluginConfigurationConverter), eq(PluginSetting.class), eq(pluginSetting)); - verify(beanFactoryProvider).createPluginSpecificContext(new Class[]{}, convertedConfiguration); + verify(beanFactoryProvider).createPluginSpecificContext(new Class[]{}, convertedConfiguration, pluginSetting); } @Test @@ -283,7 +283,7 @@ void loadPlugins_should_return_an_empty_list_when_the_number_of_instances_is_0() assertThat(plugins, notNullValue()); assertThat(plugins.size(), equalTo(0)); - verify(beanFactoryProvider).createPluginSpecificContext(new Class[]{}, null); + verify(beanFactoryProvider).createPluginSpecificContext(new Class[]{}, null, pluginSetting); verifyNoInteractions(pluginCreator); } @@ -299,7 +299,7 @@ void loadPlugins_should_return_a_single_instance_when_the_the_numberOfInstances_ final List plugins = createObjectUnderTest().loadPlugins( baseClass, pluginSetting, c -> 1); - verify(beanFactoryProvider).createPluginSpecificContext(new Class[]{}, convertedConfiguration); + verify(beanFactoryProvider).createPluginSpecificContext(new Class[]{}, convertedConfiguration, pluginSetting); verify(pluginConfigurationObservableFactory).createDefaultPluginConfigObservable(eq(pluginConfigurationConverter), eq(PluginSetting.class), eq(pluginSetting)); final ArgumentCaptor pluginArgumentsContextArgCapture = ArgumentCaptor.forClass(ComponentPluginArgumentsContext.class); @@ -309,7 +309,7 @@ void loadPlugins_should_return_a_single_instance_when_the_the_numberOfInstances_ final Object[] pipelineDescriptionObj = actualPluginArgumentsContext.createArguments(classes.toArray(new Class[1])); assertThat(pipelineDescriptionObj.length, equalTo(1)); assertThat(pipelineDescriptionObj[0], instanceOf(PipelineDescription.class)); - final PipelineDescription actualPipelineDescription = (PipelineDescription)pipelineDescriptionObj[0]; + final PipelineDescription actualPipelineDescription = (PipelineDescription) pipelineDescriptionObj[0]; assertThat(actualPipelineDescription.getPipelineName(), is(pipelineName)); assertThat(plugins, notNullValue()); assertThat(plugins.size(), equalTo(1)); @@ -328,7 +328,7 @@ void loadPlugin_with_varargs_should_return_a_single_instance_when_the_the_number final Object plugin = createObjectUnderTest().loadPlugin(baseClass, pluginSetting, object); - verify(beanFactoryProvider).createPluginSpecificContext(new Class[]{}, convertedConfiguration); + verify(beanFactoryProvider).createPluginSpecificContext(new Class[]{}, convertedConfiguration, pluginSetting); verify(pluginConfigurationObservableFactory).createDefaultPluginConfigObservable(eq(pluginConfigurationConverter), eq(PluginSetting.class), eq(pluginSetting)); final ArgumentCaptor pluginArgumentsContextArgCapture = ArgumentCaptor.forClass(ComponentPluginArgumentsContext.class); @@ -338,7 +338,7 @@ void loadPlugin_with_varargs_should_return_a_single_instance_when_the_the_number final Object[] pipelineDescriptionObj = actualPluginArgumentsContext.createArguments(classes.toArray(new Class[1])); assertThat(pipelineDescriptionObj.length, equalTo(1)); assertThat(pipelineDescriptionObj[0], instanceOf(PipelineDescription.class)); - final PipelineDescription actualPipelineDescription = (PipelineDescription)pipelineDescriptionObj[0]; + final PipelineDescription actualPipelineDescription = (PipelineDescription) pipelineDescriptionObj[0]; assertThat(actualPipelineDescription.getPipelineName(), is(pipelineName)); assertThat(plugin, notNullValue()); assertThat(plugin, equalTo(expectedInstance)); @@ -380,7 +380,7 @@ void loadPlugins_should_return_an_instance_for_the_total_count() { final List plugins = createObjectUnderTest().loadPlugins( baseClass, pluginSetting, c -> 3); - verify(beanFactoryProvider).createPluginSpecificContext(new Class[]{}, convertedConfiguration); + verify(beanFactoryProvider).createPluginSpecificContext(new Class[]{}, convertedConfiguration, pluginSetting); final ArgumentCaptor pluginArgumentsContextArgCapture = ArgumentCaptor.forClass(ComponentPluginArgumentsContext.class); verify(pluginCreator, times(3)).newPluginInstance(eq(expectedPluginClass), pluginArgumentsContextArgCapture.capture(), eq(pluginName)); final List actualPluginArgumentsContextList = pluginArgumentsContextArgCapture.getAllValues(); @@ -390,7 +390,7 @@ void loadPlugins_should_return_an_instance_for_the_total_count() { final Object[] pipelineDescriptionObj = pluginArgumentsContext.createArguments(classes.toArray(new Class[1])); assertThat(pipelineDescriptionObj.length, equalTo(1)); assertThat(pipelineDescriptionObj[0], instanceOf(PipelineDescription.class)); - final PipelineDescription actualPipelineDescription = (PipelineDescription)pipelineDescriptionObj[0]; + final PipelineDescription actualPipelineDescription = (PipelineDescription) pipelineDescriptionObj[0]; assertThat(actualPipelineDescription.getPipelineName(), is(pipelineName)); }); assertThat(plugins, notNullValue()); @@ -416,7 +416,7 @@ void loadPlugins_should_return_a_single_instance_with_values_from_ApplicationCon final List plugins = createObjectUnderTest().loadPlugins( baseClass, pluginSetting, c -> 1); - verify(beanFactoryProvider).createPluginSpecificContext(new Class[]{}, convertedConfiguration); + verify(beanFactoryProvider).createPluginSpecificContext(new Class[]{}, convertedConfiguration, pluginSetting); final ArgumentCaptor pluginArgumentsContextArgCapture = ArgumentCaptor.forClass(ComponentPluginArgumentsContext.class); verify(pluginCreator).newPluginInstance(eq(expectedPluginClass), pluginArgumentsContextArgCapture.capture(), eq(pluginName)); final ComponentPluginArgumentsContext actualPluginArgumentsContext = pluginArgumentsContextArgCapture.getValue(); @@ -425,7 +425,7 @@ void loadPlugins_should_return_a_single_instance_with_values_from_ApplicationCon assertThat(pipelineDescriptionObj.length, equalTo(2)); assertThat(pipelineDescriptionObj[0], instanceOf(PipelineDescription.class)); assertThat(pipelineDescriptionObj[1], sameInstance(suppliedAdditionalArgument)); - final PipelineDescription actualPipelineDescription = (PipelineDescription)pipelineDescriptionObj[0]; + final PipelineDescription actualPipelineDescription = (PipelineDescription) pipelineDescriptionObj[0]; assertThat(actualPipelineDescription.getPipelineName(), is(pipelineName)); assertThat(plugins, notNullValue()); assertThat(plugins.size(), equalTo(1)); @@ -458,7 +458,7 @@ void loadPlugin_should_create_a_new_instance_of_the_first_plugin_found_with_corr assertThat(createObjectUnderTest().loadPlugin(baseClass, pluginSetting), equalTo(expectedInstance)); MatcherAssert.assertThat(expectedInstance.getClass().getAnnotation(DataPrepperPlugin.class).deprecatedName(), equalTo(TEST_SINK_DEPRECATED_NAME)); - verify(beanFactoryProvider).createPluginSpecificContext(new Class[]{}, convertedConfiguration); + verify(beanFactoryProvider).createPluginSpecificContext(new Class[]{}, convertedConfiguration, pluginSetting); } } @@ -487,7 +487,7 @@ void loadPlugin_should_create_a_new_instance_of_the_first_plugin_found_with_corr assertThat(createObjectUnderTest().loadPlugin(baseClass, pluginSetting), equalTo(expectedInstance)); MatcherAssert.assertThat(expectedInstance.getClass().getAnnotation(DataPrepperPlugin.class).alternateNames(), equalTo(new String[]{TEST_SINK_ALTERNATE_NAME})); - verify(beanFactoryProvider).createPluginSpecificContext(new Class[]{}, convertedConfiguration); + verify(beanFactoryProvider).createPluginSpecificContext(new Class[]{}, convertedConfiguration, pluginSetting); } } } diff --git a/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/PluginBeanFactoryProviderTest.java b/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/PluginBeanFactoryProviderTest.java index a80e42c011..85628db855 100644 --- a/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/PluginBeanFactoryProviderTest.java +++ b/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/PluginBeanFactoryProviderTest.java @@ -78,7 +78,7 @@ void testPluginBeanFactoryProviderGetReturnsBeanFactory() { final PluginBeanFactoryProvider beanFactoryProvider = createObjectUnderTest(); verify(context).getParent(); - assertThat(beanFactoryProvider.createPluginSpecificContext(new Class[]{}, null), is(instanceOf(BeanFactory.class))); + assertThat(beanFactoryProvider.createPluginSpecificContext(new Class[]{}, null, null), is(instanceOf(BeanFactory.class))); } @Test @@ -86,8 +86,8 @@ void testPluginBeanFactoryProviderGetReturnsUniqueBeanFactory() { doReturn(context).when(context).getParent(); final PluginBeanFactoryProvider beanFactoryProvider = createObjectUnderTest(); - final BeanFactory isolatedBeanFactoryA = beanFactoryProvider.createPluginSpecificContext(new Class[]{}, null); - final BeanFactory isolatedBeanFactoryB = beanFactoryProvider.createPluginSpecificContext(new Class[]{}, null); + final BeanFactory isolatedBeanFactoryA = beanFactoryProvider.createPluginSpecificContext(new Class[]{}, null, null); + final BeanFactory isolatedBeanFactoryB = beanFactoryProvider.createPluginSpecificContext(new Class[]{}, null, null); verify(context).getParent(); assertThat(isolatedBeanFactoryA, not(sameInstance(isolatedBeanFactoryB))); @@ -113,7 +113,7 @@ void getSharedPluginApplicationContext_called_multiple_times_returns_same_instan void testCreatePluginSpecificContext() { when(context.getParent()).thenReturn(context); final PluginBeanFactoryProvider objectUnderTest = createObjectUnderTest(); - BeanFactory beanFactory = objectUnderTest.createPluginSpecificContext(new Class[]{TestComponent.class}, null); + BeanFactory beanFactory = objectUnderTest.createPluginSpecificContext(new Class[]{TestComponent.class}, null, null); assertThat(beanFactory, notNullValue()); assertThat(beanFactory.getBean(TestComponent.class), notNullValue()); } @@ -122,7 +122,7 @@ void testCreatePluginSpecificContext() { void testCreatePluginSpecificContext_with_empty_array() { when(context.getParent()).thenReturn(context); final PluginBeanFactoryProvider objectUnderTest = createObjectUnderTest(); - BeanFactory beanFactory = objectUnderTest.createPluginSpecificContext(new Class[]{}, null); + BeanFactory beanFactory = objectUnderTest.createPluginSpecificContext(new Class[]{}, null, null); assertThat(beanFactory, notNullValue()); assertThat(beanFactory, instanceOf(ListableBeanFactory.class)); ListableBeanFactory listableBeanFactory = (ListableBeanFactory) beanFactory; @@ -137,9 +137,9 @@ void testCreatePluginSpecificContext_with_pipeline_settings() { when(context.getParent()).thenReturn(context); final PluginBeanFactoryProvider objectUnderTest = createObjectUnderTest(); PluginSetting pipelineSettings = new PluginSetting(UUID.randomUUID().toString(), Map.of("key", "val")); - BeanFactory beanFactory = objectUnderTest.createPluginSpecificContext(new Class[]{}, pipelineSettings); + BeanFactory beanFactory = objectUnderTest.createPluginSpecificContext(new Class[]{}, pipelineSettings, pipelineSettings); assertThat(beanFactory, notNullValue()); - assertThrows(NoSuchBeanDefinitionException.class, ()->beanFactory.getBean(PluginSetting.class)); + assertThrows(NoSuchBeanDefinitionException.class, () -> beanFactory.getBean(PluginSetting.class)); } @Test @@ -147,10 +147,10 @@ void testCreatePluginSpecificContext_with_empty_array_with_plugin_config() { when(context.getParent()).thenReturn(context); final PluginBeanFactoryProvider objectUnderTest = createObjectUnderTest(); TestPluginConfiguration config = new TestPluginConfiguration(); - BeanFactory beanFactory = objectUnderTest.createPluginSpecificContext(new Class[]{}, config); + BeanFactory beanFactory = objectUnderTest.createPluginSpecificContext(new Class[]{}, config, null); assertThat(beanFactory, notNullValue()); - assertThrows(NoSuchBeanDefinitionException.class, ()->beanFactory.getBean(TestComponent.class)); - assertThrows(NoSuchBeanDefinitionException.class, ()->beanFactory.getBean(TestPluginConfiguration.class)); + assertThrows(NoSuchBeanDefinitionException.class, () -> beanFactory.getBean(TestComponent.class)); + assertThrows(NoSuchBeanDefinitionException.class, () -> beanFactory.getBean(TestPluginConfiguration.class)); } @Test @@ -160,7 +160,7 @@ void testCreatePluginSpecificContext_with_plugin_config() { TestPluginConfiguration config = new TestPluginConfiguration(); String requiredStringValue = UUID.randomUUID().toString(); config.setRequiredString(requiredStringValue); - BeanFactory beanFactory = objectUnderTest.createPluginSpecificContext(new Class[]{TestComponent.class}, config); + BeanFactory beanFactory = objectUnderTest.createPluginSpecificContext(new Class[]{TestComponent.class}, config, null); assertThat(beanFactory, notNullValue()); assertThat(beanFactory.getBean(TestComponent.class), notNullValue()); assertThat(beanFactory.getBean(TestPluginConfiguration.class), notNullValue()); diff --git a/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugins/configtest/TestComponentWithConfigInject.java b/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugins/configtest/TestComponentWithConfigInject.java index 141f4f5264..21d95b5808 100644 --- a/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugins/configtest/TestComponentWithConfigInject.java +++ b/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugins/configtest/TestComponentWithConfigInject.java @@ -1,5 +1,6 @@ package org.opensearch.dataprepper.plugins.configtest; +import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.plugin.TestPluginConfiguration; import javax.inject.Named; @@ -7,9 +8,11 @@ @Named public class TestComponentWithConfigInject { private final TestPluginConfiguration configuration; + private final PluginMetrics pluginMetrics; - public TestComponentWithConfigInject(TestPluginConfiguration configuration) { + public TestComponentWithConfigInject(TestPluginConfiguration configuration, PluginMetrics pluginMetrics) { this.configuration = configuration; + this.pluginMetrics = pluginMetrics; } public String getIdentifier() { @@ -19,4 +22,8 @@ public String getIdentifier() { public TestPluginConfiguration getConfiguration() { return configuration; } + + public PluginMetrics getPluginMetrics() { + return pluginMetrics; + } } diff --git a/data-prepper-plugins/otel-trace-raw-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/oteltrace/OTelTraceRawProcessorTest.java b/data-prepper-plugins/otel-trace-raw-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/oteltrace/OTelTraceRawProcessorTest.java index 9163aad322..f934bc2a4c 100644 --- a/data-prepper-plugins/otel-trace-raw-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/oteltrace/OTelTraceRawProcessorTest.java +++ b/data-prepper-plugins/otel-trace-raw-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/oteltrace/OTelTraceRawProcessorTest.java @@ -255,7 +255,7 @@ void testMetricsOnSpanSet() { "0, 4", "2, 6" }) - void traceGroupCacheMaxSize_provides_an_upper_bound(final long cacheMaxSize, final int expectedProcessedRecords) { + void traceGroupCacheMaxSize_provides_an_upper_bound(final long cacheMaxSize, final int leastExpectedProcessedRecordCount) { reset(config); when(config.getTraceFlushIntervalSeconds()).thenReturn(TEST_TRACE_FLUSH_INTERVAL); when(config.getTraceGroupCacheMaxSize()).thenReturn(cacheMaxSize); @@ -267,7 +267,12 @@ void traceGroupCacheMaxSize_provides_an_upper_bound(final long cacheMaxSize, fin processedRecords.addAll(oTelTraceRawProcessor.doExecute(TEST_TWO_TRACE_GROUP_INTERLEAVED_PART_1_RECORDS)); processedRecords.addAll(oTelTraceRawProcessor.doExecute(TEST_TWO_TRACE_GROUP_INTERLEAVED_PART_2_RECORDS)); - MatcherAssert.assertThat(processedRecords.size(), equalTo(expectedProcessedRecords)); + // Caffeine cache eviction is not a synchronized action so trying to make an exact match with the + // expected record count may not always match depending on when the records are getting evicted. + // So changing this assertion from equals to greater than equals to expected record count + assertTrue(processedRecords.size() >= leastExpectedProcessedRecordCount, + String.format("Processed records of %d should be at least the expected record count of %d", + processedRecords.size(), leastExpectedProcessedRecordCount)); MatcherAssert.assertThat(getMissingTraceGroupFieldsSpanCount(processedRecords), equalTo(0)); } diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraService.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraService.java index 415c4d54c8..46e51fcbaa 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraService.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraService.java @@ -61,13 +61,12 @@ public class JiraService { private final JiraSourceConfig jiraSourceConfig; private final JiraRestClient jiraRestClient; private final Counter searchResultsFoundCounter; - private final PluginMetrics jiraPluginMetrics = PluginMetrics.fromNames("jiraService", "aws"); - public JiraService(JiraSourceConfig jiraSourceConfig, JiraRestClient jiraRestClient) { + public JiraService(JiraSourceConfig jiraSourceConfig, JiraRestClient jiraRestClient, PluginMetrics pluginMetrics) { this.jiraSourceConfig = jiraSourceConfig; this.jiraRestClient = jiraRestClient; - this.searchResultsFoundCounter = jiraPluginMetrics.counter(SEARCH_RESULTS_FOUND); + this.searchResultsFoundCounter = pluginMetrics.counter(SEARCH_RESULTS_FOUND); } /** @@ -99,7 +98,7 @@ private void searchForNewTicketsAndAddToQueue(JiraSourceConfig configuration, In int total; int startAt = 0; do { - SearchResults searchIssues = jiraRestClient.getAllIssues(jql, startAt, configuration); + SearchResults searchIssues = jiraRestClient.getAllIssues(jql, startAt); List issueList = new ArrayList<>(searchIssues.getIssues()); total = searchIssues.getTotal(); startAt += searchIssues.getIssues().size(); @@ -132,7 +131,7 @@ private void addItemsToQueue(List issueList, Queue itemInfo private StringBuilder createIssueFilterCriteria(JiraSourceConfig configuration, Instant ts) { log.info("Creating issue filter criteria"); - if (!CollectionUtils.isEmpty(JiraConfigHelper.getProjectNameIncludeFilter(configuration)) || !CollectionUtils.isEmpty(JiraConfigHelper.getProjectNameExcludeFilter(configuration)) ) { + if (!CollectionUtils.isEmpty(JiraConfigHelper.getProjectNameIncludeFilter(configuration)) || !CollectionUtils.isEmpty(JiraConfigHelper.getProjectNameExcludeFilter(configuration))) { validateProjectFilters(configuration); } StringBuilder jiraQl = new StringBuilder(UPDATED + GREATER_THAN_EQUALS + ts.toEpochMilli()); diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java index 28a71f55ee..5271c6cfb5 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java @@ -16,7 +16,6 @@ import io.micrometer.core.instrument.Timer; import lombok.extern.slf4j.Slf4j; import org.opensearch.dataprepper.metrics.PluginMetrics; -import org.opensearch.dataprepper.plugins.source.jira.JiraSourceConfig; import org.opensearch.dataprepper.plugins.source.jira.exception.BadRequestException; import org.opensearch.dataprepper.plugins.source.jira.exception.UnAuthorizedException; import org.opensearch.dataprepper.plugins.source.jira.models.SearchResults; @@ -44,42 +43,42 @@ public class JiraRestClient { public static final String REST_API_SEARCH = "rest/api/3/search"; public static final String REST_API_FETCH_ISSUE = "rest/api/3/issue"; - public static final String REST_API_PROJECTS = "/rest/api/3/project/search"; + //public static final String REST_API_PROJECTS = "/rest/api/3/project/search"; public static final String FIFTY = "50"; public static final String START_AT = "startAt"; public static final String MAX_RESULT = "maxResults"; public static final List RETRY_ATTEMPT_SLEEP_TIME = List.of(1, 2, 5, 10, 20, 40); private static final String TICKET_FETCH_LATENCY_TIMER = "ticketFetchLatency"; private static final String SEARCH_CALL_LATENCY_TIMER = "searchCallLatency"; + private static final String PROJECTS_FETCH_LATENCY_TIMER = "projectFetchLatency"; private static final String ISSUES_REQUESTED = "issuesRequested"; + private int sleepTimeMultiplier = 1000; private final RestTemplate restTemplate; private final JiraAuthConfig authConfig; private final Timer ticketFetchLatencyTimer; private final Timer searchCallLatencyTimer; + private final Timer projectFetchLatencyTimer; private final Counter issuesRequestedCounter; - private final PluginMetrics jiraPluginMetrics = PluginMetrics.fromNames("jiraRestClient", "aws"); - private int sleepTimeMultiplier = 1000; - public JiraRestClient(RestTemplate restTemplate, JiraAuthConfig authConfig) { + public JiraRestClient(RestTemplate restTemplate, JiraAuthConfig authConfig, PluginMetrics pluginMetrics) { this.restTemplate = restTemplate; this.authConfig = authConfig; - ticketFetchLatencyTimer = jiraPluginMetrics.timer(TICKET_FETCH_LATENCY_TIMER); - searchCallLatencyTimer = jiraPluginMetrics.timer(SEARCH_CALL_LATENCY_TIMER); - issuesRequestedCounter = jiraPluginMetrics.counter(ISSUES_REQUESTED); + ticketFetchLatencyTimer = pluginMetrics.timer(TICKET_FETCH_LATENCY_TIMER); + searchCallLatencyTimer = pluginMetrics.timer(SEARCH_CALL_LATENCY_TIMER); + projectFetchLatencyTimer = pluginMetrics.timer(PROJECTS_FETCH_LATENCY_TIMER); + issuesRequestedCounter = pluginMetrics.counter(ISSUES_REQUESTED); } /** * Method to get Issues. * - * @param jql input parameter. - * @param startAt the start at - * @param configuration input parameter. + * @param jql input parameter. + * @param startAt the start at * @return InputStream input stream */ @Timed(SEARCH_CALL_LATENCY_TIMER) - public SearchResults getAllIssues(StringBuilder jql, int startAt, - JiraSourceConfig configuration) { + public SearchResults getAllIssues(StringBuilder jql, int startAt) { String url = authConfig.getUrl() + REST_API_SEARCH; diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraIteratorTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraIteratorTest.java index 73e184330a..bd73e3e966 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraIteratorTest.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraIteratorTest.java @@ -15,6 +15,7 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.plugins.source.jira.models.IssueBean; import org.opensearch.dataprepper.plugins.source.jira.models.SearchResults; import org.opensearch.dataprepper.plugins.source.jira.rest.JiraRestClient; @@ -45,8 +46,6 @@ @ExtendWith(MockitoExtension.class) public class JiraIteratorTest { - private final PluginExecutorServiceProvider executorServiceProvider = new PluginExecutorServiceProvider(); - @Mock private SearchResults mockSearchResults; @Mock @@ -54,12 +53,13 @@ public class JiraIteratorTest { private JiraService jiraService; @Mock private JiraSourceConfig jiraSourceConfig; - private JiraIterator jiraIterator; + private final PluginExecutorServiceProvider executorServiceProvider = new PluginExecutorServiceProvider(); @BeforeEach void setUp() { - jiraService = spy(new JiraService(jiraSourceConfig, jiraRestClient)); + jiraService = spy(new JiraService(jiraSourceConfig, jiraRestClient, + PluginMetrics.fromNames("jiraIteratorTest", "jira"))); } public JiraIterator createObjectUnderTest() { @@ -73,7 +73,7 @@ void testInitialization() { jiraIterator.initialize(Instant.ofEpochSecond(0)); when(mockSearchResults.getIssues()).thenReturn(new ArrayList<>()); when(mockSearchResults.getTotal()).thenReturn(0); - doReturn(mockSearchResults).when(jiraRestClient).getAllIssues(any(StringBuilder.class), anyInt(), any(JiraSourceConfig.class)); + doReturn(mockSearchResults).when(jiraRestClient).getAllIssues(any(StringBuilder.class), anyInt()); assertFalse(jiraIterator.hasNext()); } @@ -104,7 +104,7 @@ void testItemInfoQueueNotEmpty() { mockIssues.add(issue1); when(mockSearchResults.getIssues()).thenReturn(mockIssues); when(mockSearchResults.getTotal()).thenReturn(0); - doReturn(mockSearchResults).when(jiraRestClient).getAllIssues(any(StringBuilder.class), anyInt(), any(JiraSourceConfig.class)); + doReturn(mockSearchResults).when(jiraRestClient).getAllIssues(any(StringBuilder.class), anyInt()); jiraIterator.initialize(Instant.ofEpochSecond(0)); jiraIterator.setCrawlerQWaitTimeMillis(1); @@ -118,7 +118,7 @@ void testStartCrawlerThreads() { jiraIterator.initialize(Instant.ofEpochSecond(0)); jiraIterator.hasNext(); jiraIterator.hasNext(); - assertTrue(jiraIterator.showFutureList().size() == 1); + assertEquals(1, jiraIterator.showFutureList().size()); } @Test @@ -133,7 +133,7 @@ void testFuturesCompleted() throws InterruptedException { mockIssues.add(issue3); when(mockSearchResults.getIssues()).thenReturn(mockIssues); when(mockSearchResults.getTotal()).thenReturn(0); - doReturn(mockSearchResults).when(jiraRestClient).getAllIssues(any(StringBuilder.class), anyInt(), any(JiraSourceConfig.class)); + doReturn(mockSearchResults).when(jiraRestClient).getAllIssues(any(StringBuilder.class), anyInt()); jiraIterator.initialize(Instant.ofEpochSecond(0)); jiraIterator.setCrawlerQWaitTimeMillis(1); @@ -145,12 +145,12 @@ void testFuturesCompleted() throws InterruptedException { } @Test - void testItemInfoQueueEmpty(){ + void testItemInfoQueueEmpty() { jiraIterator = createObjectUnderTest(); List mockIssues = new ArrayList<>(); when(mockSearchResults.getIssues()).thenReturn(mockIssues); when(mockSearchResults.getTotal()).thenReturn(0); - doReturn(mockSearchResults).when(jiraRestClient).getAllIssues(any(StringBuilder.class), anyInt(), any(JiraSourceConfig.class)); + doReturn(mockSearchResults).when(jiraRestClient).getAllIssues(any(StringBuilder.class), anyInt()); jiraIterator.initialize(Instant.ofEpochSecond(0)); jiraIterator.setCrawlerQWaitTimeMillis(1); diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraServiceTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraServiceTest.java index 484ca2caac..80b5c50c71 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraServiceTest.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraServiceTest.java @@ -18,6 +18,7 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.plugin.PluginConfigVariable; import org.opensearch.dataprepper.plugins.source.jira.configuration.Oauth2Config; import org.opensearch.dataprepper.plugins.source.jira.exception.BadRequestException; @@ -71,11 +72,10 @@ public class JiraServiceTest { private static final Logger log = LoggerFactory.getLogger(JiraServiceTest.class); - private final PluginExecutorServiceProvider executorServiceProvider = new PluginExecutorServiceProvider(); - @Mock private JiraRestClient jiraRestClient; - + private final PluginExecutorServiceProvider executorServiceProvider = new PluginExecutorServiceProvider(); + private final PluginMetrics pluginMetrics = PluginMetrics.fromNames("JiraServiceTest", "jira"); private static InputStream getResourceAsStream(String resourceName) { InputStream inputStream = Thread.currentThread().getContextClassLoader().getResourceAsStream(resourceName); @@ -178,7 +178,7 @@ void testJiraServiceInitialization() throws JsonProcessingException { List issueStatus = new ArrayList<>(); List projectKey = new ArrayList<>(); JiraSourceConfig jiraSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey); - JiraService jiraService = new JiraService(jiraSourceConfig, jiraRestClient); + JiraService jiraService = new JiraService(jiraSourceConfig, jiraRestClient, pluginMetrics); assertNotNull(jiraService); when(jiraRestClient.getIssue(anyString())).thenReturn("test String"); assertNotNull(jiraService.getIssue("test Key")); @@ -193,7 +193,7 @@ public void testGetJiraEntities() throws JsonProcessingException { issueStatus.add("Done"); projectKey.add("KAN"); JiraSourceConfig jiraSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey); - JiraService jiraService = spy(new JiraService(jiraSourceConfig, jiraRestClient)); + JiraService jiraService = spy(new JiraService(jiraSourceConfig, jiraRestClient, pluginMetrics)); List mockIssues = new ArrayList<>(); IssueBean issue1 = createIssueBean(false, false); mockIssues.add(issue1); @@ -206,7 +206,7 @@ public void testGetJiraEntities() throws JsonProcessingException { when(mockSearchResults.getIssues()).thenReturn(mockIssues); when(mockSearchResults.getTotal()).thenReturn(mockIssues.size()); - doReturn(mockSearchResults).when(jiraRestClient).getAllIssues(any(StringBuilder.class), anyInt(), any(JiraSourceConfig.class)); + doReturn(mockSearchResults).when(jiraRestClient).getAllIssues(any(StringBuilder.class), anyInt()); Instant timestamp = Instant.ofEpochSecond(0); Queue itemInfoQueue = new ConcurrentLinkedQueue<>(); @@ -221,7 +221,7 @@ public void buildIssueItemInfoMultipleFutureThreads() throws JsonProcessingExcep List projectKey = new ArrayList<>(); issueType.add("Task"); JiraSourceConfig jiraSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey); - JiraService jiraService = spy(new JiraService(jiraSourceConfig, jiraRestClient)); + JiraService jiraService = spy(new JiraService(jiraSourceConfig, jiraRestClient, pluginMetrics)); List mockIssues = new ArrayList<>(); for (int i = 0; i < 50; i++) { IssueBean issue1 = createIssueBean(false, false); @@ -232,7 +232,7 @@ public void buildIssueItemInfoMultipleFutureThreads() throws JsonProcessingExcep when(mockSearchResults.getIssues()).thenReturn(mockIssues); when(mockSearchResults.getTotal()).thenReturn(100); - doReturn(mockSearchResults).when(jiraRestClient).getAllIssues(any(StringBuilder.class), anyInt(), any(JiraSourceConfig.class)); + doReturn(mockSearchResults).when(jiraRestClient).getAllIssues(any(StringBuilder.class), anyInt()); Instant timestamp = Instant.ofEpochSecond(0); Queue itemInfoQueue = new ConcurrentLinkedQueue<>(); @@ -253,7 +253,7 @@ public void testBadProjectKeys() throws JsonProcessingException { projectKey.add("AAAAAAAAAAAAAA"); JiraSourceConfig jiraSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey); - JiraService jiraService = new JiraService(jiraSourceConfig, jiraRestClient); + JiraService jiraService = new JiraService(jiraSourceConfig, jiraRestClient, pluginMetrics); Instant timestamp = Instant.ofEpochSecond(0); Queue itemInfoQueue = new ConcurrentLinkedQueue<>(); @@ -268,9 +268,9 @@ public void testGetJiraEntitiesException() throws JsonProcessingException { List projectKey = new ArrayList<>(); issueType.add("Task"); JiraSourceConfig jiraSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey); - JiraService jiraService = spy(new JiraService(jiraSourceConfig, jiraRestClient)); + JiraService jiraService = spy(new JiraService(jiraSourceConfig, jiraRestClient, pluginMetrics)); - doThrow(RuntimeException.class).when(jiraRestClient).getAllIssues(any(StringBuilder.class), anyInt(), any(JiraSourceConfig.class)); + doThrow(RuntimeException.class).when(jiraRestClient).getAllIssues(any(StringBuilder.class), anyInt()); Instant timestamp = Instant.ofEpochSecond(0); Queue itemInfoQueue = new ConcurrentLinkedQueue<>(); diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClientTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClientTest.java index 3fe0640700..a24b582e97 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClientTest.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClientTest.java @@ -19,6 +19,7 @@ import org.junit.jupiter.params.provider.ValueSource; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.plugins.source.jira.JiraServiceTest; import org.opensearch.dataprepper.plugins.source.jira.JiraSourceConfig; import org.opensearch.dataprepper.plugins.source.jira.exception.BadRequestException; @@ -43,8 +44,6 @@ import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.BASIC; -import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.OAUTH2; @ExtendWith(MockitoExtension.class) public class JiraRestClientTest { @@ -57,6 +56,7 @@ public class JiraRestClientTest { @Mock private JiraAuthConfig authConfig; + private final PluginMetrics pluginMetrics = PluginMetrics.fromNames("JiraRestClientTest", "jira"); private static Stream provideHttpStatusCodesWithExceptionClass() { return Stream.of( @@ -74,7 +74,7 @@ public void testFetchingJiraIssue(String configFileName) { doReturn(new ResponseEntity<>(exampleTicketResponse, HttpStatus.OK)).when(restTemplate).getForEntity(any(URI.class), any(Class.class)); JiraSourceConfig jiraSourceConfig = JiraServiceTest.createJiraConfigurationFromYaml(configFileName); JiraAuthConfig authConfig = new JiraAuthFactory(jiraSourceConfig).getObject(); - JiraRestClient jiraRestClient = new JiraRestClient(restTemplate, authConfig); + JiraRestClient jiraRestClient = new JiraRestClient(restTemplate, authConfig, pluginMetrics); String ticketDetails = jiraRestClient.getIssue("key"); assertEquals(exampleTicketResponse, ticketDetails); } @@ -82,7 +82,7 @@ public void testFetchingJiraIssue(String configFileName) { @ParameterizedTest @MethodSource("provideHttpStatusCodesWithExceptionClass") void testInvokeRestApiTokenExpired(HttpStatus statusCode, Class expectedExceptionType) { - JiraRestClient jiraRestClient = new JiraRestClient(restTemplate, authConfig); + JiraRestClient jiraRestClient = new JiraRestClient(restTemplate, authConfig, pluginMetrics); jiraRestClient.setSleepTimeMultiplier(1); when(authConfig.getUrl()).thenReturn("https://example.com/rest/api/2/issue/key"); when(restTemplate.getForEntity(any(URI.class), any(Class.class))).thenThrow(new HttpClientErrorException(statusCode)); @@ -91,7 +91,7 @@ void testInvokeRestApiTokenExpired(HttpStatus statusCode, Class expectedExceptio @Test void testInvokeRestApiTokenExpiredInterruptException() throws InterruptedException { - JiraRestClient jiraRestClient = new JiraRestClient(restTemplate, authConfig); + JiraRestClient jiraRestClient = new JiraRestClient(restTemplate, authConfig, pluginMetrics); when(authConfig.getUrl()).thenReturn("https://example.com/rest/api/2/issue/key"); when(restTemplate.getForEntity(any(URI.class), any(Class.class))).thenThrow(new HttpClientErrorException(HttpStatus.TOO_MANY_REQUESTS)); jiraRestClient.setSleepTimeMultiplier(100000); @@ -111,39 +111,33 @@ void testInvokeRestApiTokenExpiredInterruptException() throws InterruptedExcepti } @Test - public void testGetAllIssuesOauth2() throws JsonProcessingException { + public void testGetAllIssuesOauth2() { List issueType = new ArrayList<>(); - List issueStatus = new ArrayList<>(); - List projectKey = new ArrayList<>(); issueType.add("Task"); - JiraSourceConfig jiraSourceConfig = JiraServiceTest.createJiraConfiguration(OAUTH2, issueType, issueStatus, projectKey); - JiraRestClient jiraRestClient = new JiraRestClient(restTemplate, authConfig); + JiraRestClient jiraRestClient = new JiraRestClient(restTemplate, authConfig, pluginMetrics); SearchResults mockSearchResults = mock(SearchResults.class); doReturn("http://mock-service.jira.com/").when(authConfig).getUrl(); doReturn(new ResponseEntity<>(mockSearchResults, HttpStatus.OK)).when(restTemplate).getForEntity(any(URI.class), any(Class.class)); - SearchResults results = jiraRestClient.getAllIssues(jql, 0, jiraSourceConfig); + SearchResults results = jiraRestClient.getAllIssues(jql, 0); assertNotNull(results); } @Test - public void testGetAllIssuesBasic() throws JsonProcessingException { + public void testGetAllIssuesBasic() { List issueType = new ArrayList<>(); - List issueStatus = new ArrayList<>(); - List projectKey = new ArrayList<>(); issueType.add("Task"); - JiraSourceConfig jiraSourceConfig = JiraServiceTest.createJiraConfiguration(BASIC, issueType, issueStatus, projectKey); - JiraRestClient jiraRestClient = new JiraRestClient(restTemplate, authConfig); + JiraRestClient jiraRestClient = new JiraRestClient(restTemplate, authConfig, pluginMetrics); SearchResults mockSearchResults = mock(SearchResults.class); when(authConfig.getUrl()).thenReturn("https://example.com/"); doReturn(new ResponseEntity<>(mockSearchResults, HttpStatus.OK)).when(restTemplate).getForEntity(any(URI.class), any(Class.class)); - SearchResults results = jiraRestClient.getAllIssues(jql, 0, jiraSourceConfig); + SearchResults results = jiraRestClient.getAllIssues(jql, 0); assertNotNull(results); } @Test public void testRestApiAddressValidation() throws JsonProcessingException { when(authConfig.getUrl()).thenReturn("https://224.0.0.1/"); - JiraRestClient jiraRestClient = new JiraRestClient(restTemplate, authConfig); + JiraRestClient jiraRestClient = new JiraRestClient(restTemplate, authConfig, pluginMetrics); assertThrows(BadRequestException.class, () -> jiraRestClient.getIssue("TEST-1")); } diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/base/Crawler.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/base/Crawler.java index 7a8c4f2275..d93461ee79 100644 --- a/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/base/Crawler.java +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/base/Crawler.java @@ -30,12 +30,10 @@ public class Crawler { private static final Logger log = LoggerFactory.getLogger(Crawler.class); private final Timer crawlingTimer; - private final PluginMetrics pluginMetrics = - PluginMetrics.fromNames("sourceCrawler", "crawler"); private final CrawlerClient client; - public Crawler(CrawlerClient client) { + public Crawler(CrawlerClient client, PluginMetrics pluginMetrics) { this.client = client; this.crawlingTimer = pluginMetrics.timer("crawlingTime"); } diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/coordination/scheduler/WorkerScheduler.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/coordination/scheduler/WorkerScheduler.java index e738c0e19c..6a6fbd2471 100644 --- a/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/coordination/scheduler/WorkerScheduler.java +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/coordination/scheduler/WorkerScheduler.java @@ -40,7 +40,6 @@ public class WorkerScheduler implements Runnable { private final Counter acknowledgementSetSuccesses; private final Counter acknowledgementSetFailures; private final String sourcePluginName; - private final String SOURCE_PLUGIN_NAME = "sourcePluginName"; public WorkerScheduler(final String sourcePluginName, @@ -58,8 +57,8 @@ public WorkerScheduler(final String sourcePluginName, this.acknowledgementSetManager = acknowledgementSetManager; this.pluginMetrics = pluginMetrics; - this.acknowledgementSetSuccesses = pluginMetrics.counterWithTags(ACKNOWLEDGEMENT_SET_SUCCESS_METRIC_NAME, SOURCE_PLUGIN_NAME, sourcePluginName); - this.acknowledgementSetFailures = pluginMetrics.counterWithTags(ACKNOWLEDGEMENT_SET_FAILURES_METRIC_NAME, SOURCE_PLUGIN_NAME, sourcePluginName); + this.acknowledgementSetSuccesses = pluginMetrics.counter(ACKNOWLEDGEMENT_SET_SUCCESS_METRIC_NAME); + this.acknowledgementSetFailures = pluginMetrics.counter(ACKNOWLEDGEMENT_SET_FAILURES_METRIC_NAME); } @Override diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/base/CrawlerTest.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/base/CrawlerTest.java index d88cd7aef1..2010a5c3b4 100644 --- a/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/base/CrawlerTest.java +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/base/CrawlerTest.java @@ -5,6 +5,7 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSet; import org.opensearch.dataprepper.model.buffer.Buffer; import org.opensearch.dataprepper.model.event.Event; @@ -50,10 +51,11 @@ public class CrawlerTest { private LeaderPartition leaderPartition; private Crawler crawler; private final Instant lastPollTime = Instant.ofEpochMilli(0); + private final PluginMetrics pluginMetrics = PluginMetrics.fromNames("CrawlerTest", "crawler"); @BeforeEach public void setup() { - crawler = new Crawler(client); + crawler = new Crawler(client, pluginMetrics); when(leaderPartition.getProgressState()).thenReturn(Optional.of(new LeaderProgressState(lastPollTime))); }