From 0e89bf00c233d7195aa3fa4e10cb37da11370b3a Mon Sep 17 00:00:00 2001 From: Santhosh Gandhe <1909520+san81@users.noreply.github.com> Date: Mon, 3 Feb 2025 15:06:06 -0800 Subject: [PATCH] functional with making use of Atlassian Commons package Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com> --- .../AtlassianSourceConfig.java | 3 +- .../atlassian/rest/BasicAuthInterceptor.java | 2 +- .../rest/CustomRestTemplateConfig.java | 2 +- .../rest/auth/AtlassianAuthFactory.java | 2 +- .../rest/auth/AtlassianBasicAuthConfig.java | 2 +- .../rest/auth/AtlassianOauthConfig.java | 2 +- .../confluence-source/build.gradle | 1 + .../source/confluence/ConfluenceItemInfo.java | 2 +- .../source/confluence/ConfluenceService.java | 52 ++--- .../source/confluence/ConfluenceSource.java | 17 +- .../confluence/ConfluenceSourceConfig.java | 37 +--- .../configuration/AuthenticationConfig.java | 45 ----- .../confluence/configuration/BasicConfig.java | 29 --- .../configuration/Oauth2Config.java | 36 ---- .../confluence/rest/BasicAuthInterceptor.java | 42 ---- .../confluence/rest/ConfluenceRestClient.java | 50 +---- .../rest/CustomRestTemplateConfig.java | 42 ---- .../rest/OAuth2RequestInterceptor.java | 36 ---- .../rest/auth/ConfluenceAuthConfig.java | 34 ---- .../rest/auth/ConfluenceAuthFactory.java | 41 ---- .../rest/auth/ConfluenceBasicAuthConfig.java | 45 ----- .../rest/auth/ConfluenceOauthConfig.java | 183 ------------------ .../source/confluence/utils/Constants.java | 1 - .../rest/ConfluenceRestClientTest.java | 12 +- 24 files changed, 58 insertions(+), 660 deletions(-) rename data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/{configuration => }/AtlassianSourceConfig.java (89%) delete mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/AuthenticationConfig.java delete mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/BasicConfig.java delete mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/Oauth2Config.java delete mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/BasicAuthInterceptor.java delete mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/CustomRestTemplateConfig.java delete mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/OAuth2RequestInterceptor.java delete mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceAuthConfig.java delete mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceAuthFactory.java delete mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceBasicAuthConfig.java delete mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceOauthConfig.java diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/configuration/AtlassianSourceConfig.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/AtlassianSourceConfig.java similarity index 89% rename from data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/configuration/AtlassianSourceConfig.java rename to data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/AtlassianSourceConfig.java index 448206469a..bc661f96af 100644 --- a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/configuration/AtlassianSourceConfig.java +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/AtlassianSourceConfig.java @@ -8,11 +8,12 @@ * */ -package org.opensearch.dataprepper.plugins.source.atlassian.configuration; +package org.opensearch.dataprepper.plugins.source.atlassian; import com.fasterxml.jackson.annotation.JsonProperty; import jakarta.validation.Valid; import lombok.Getter; +import org.opensearch.dataprepper.plugins.source.atlassian.configuration.AuthenticationConfig; import org.opensearch.dataprepper.plugins.source.source_crawler.base.CrawlerSourceConfig; import java.util.List; diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/BasicAuthInterceptor.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/BasicAuthInterceptor.java index ec37d6efd3..80c5e9729b 100644 --- a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/BasicAuthInterceptor.java +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/BasicAuthInterceptor.java @@ -10,7 +10,7 @@ package org.opensearch.dataprepper.plugins.source.atlassian.rest; -import org.opensearch.dataprepper.plugins.source.atlassian.configuration.AtlassianSourceConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpRequest; import org.springframework.http.client.ClientHttpRequestExecution; diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/CustomRestTemplateConfig.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/CustomRestTemplateConfig.java index eb78131f9f..a8cd1c131d 100644 --- a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/CustomRestTemplateConfig.java +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/CustomRestTemplateConfig.java @@ -11,7 +11,7 @@ package org.opensearch.dataprepper.plugins.source.atlassian.rest; -import org.opensearch.dataprepper.plugins.source.atlassian.configuration.AtlassianSourceConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig; import org.opensearch.dataprepper.plugins.source.atlassian.rest.auth.AtlassianAuthConfig; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianAuthFactory.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianAuthFactory.java index 690f7663b2..e10f2e3d2a 100644 --- a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianAuthFactory.java +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianAuthFactory.java @@ -10,7 +10,7 @@ package org.opensearch.dataprepper.plugins.source.atlassian.rest.auth; -import org.opensearch.dataprepper.plugins.source.atlassian.configuration.AtlassianSourceConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig; import org.springframework.beans.factory.FactoryBean; import org.springframework.context.annotation.Configuration; diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianBasicAuthConfig.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianBasicAuthConfig.java index 58819d3d72..68d81bd887 100644 --- a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianBasicAuthConfig.java +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianBasicAuthConfig.java @@ -11,7 +11,7 @@ package org.opensearch.dataprepper.plugins.source.atlassian.rest.auth; -import org.opensearch.dataprepper.plugins.source.atlassian.configuration.AtlassianSourceConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig; public class AtlassianBasicAuthConfig implements AtlassianAuthConfig { diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianOauthConfig.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianOauthConfig.java index 10c11a3c7b..654785c684 100644 --- a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianOauthConfig.java +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianOauthConfig.java @@ -11,7 +11,7 @@ package org.opensearch.dataprepper.plugins.source.atlassian.rest.auth; import lombok.Getter; -import org.opensearch.dataprepper.plugins.source.atlassian.configuration.AtlassianSourceConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig; import org.opensearch.dataprepper.plugins.source.atlassian.configuration.Oauth2Config; import org.opensearch.dataprepper.plugins.source.source_crawler.exception.UnAuthorizedException; import org.slf4j.Logger; diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/build.gradle b/data-prepper-plugins/saas-source-plugins/confluence-source/build.gradle index b59ab8d251..07308a50f7 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/build.gradle +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/build.gradle @@ -5,6 +5,7 @@ plugins { dependencies { implementation project(path: ':data-prepper-plugins:saas-source-plugins:source-crawler') + implementation project(path: ':data-prepper-plugins:saas-source-plugins:atlassian-commons') implementation project(path: ':data-prepper-api') implementation project(path: ':data-prepper-plugins:aws-plugin-api') implementation project(path: ':data-prepper-plugins:buffer-common') diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceItemInfo.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceItemInfo.java index c6048caf65..f017f2588d 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceItemInfo.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceItemInfo.java @@ -137,7 +137,7 @@ public ConfluenceItemInfoBuilder withSpace(String space) { return this; } - public ConfluenceItemInfoBuilder withIssueBean(ConfluenceItem contentItem) { + public ConfluenceItemInfoBuilder withContentBean(ConfluenceItem contentItem) { Map contentItemMetadata = new HashMap<>(); contentItemMetadata.put(SPACE_KEY, contentItem.getSpaceItem().getKey()); contentItemMetadata.put(CONTENT_TITLE, contentItem.getTitle()); diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java index bc72bf3496..5893e7065d 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java @@ -48,7 +48,7 @@ /** - * Service class for interactive external Atlassian jira SaaS service and fetch required details using their rest apis. + * Service class for interactive external Atlassian Confluence SaaS service and fetch required details using their rest apis. */ @Slf4j @@ -73,7 +73,7 @@ public ConfluenceService(ConfluenceSourceConfig confluenceSourceConfig, } /** - * Get jira entities. + * Get Confluence entities. * * @param configuration the configuration. * @param timestamp timestamp. @@ -89,7 +89,7 @@ public String getContent(String contentId) { } /** - * Method for building Issue Item Info. + * Method for building Content Item Info. * * @param configuration Input Parameter * @param timestamp Input Parameter @@ -101,11 +101,11 @@ private void searchForNewContentAndAddToQueue(ConfluenceSourceConfig configurati int total; int startAt = 0; do { - ConfluenceSearchResults searchIssues = confluenceRestClient.getAllContent(cql, startAt); - List issueList = new ArrayList<>(searchIssues.getResults()); - total = searchIssues.getSize(); - startAt += searchIssues.getResults().size(); - addItemsToQueue(issueList, itemInfoQueue); + ConfluenceSearchResults searchContentItems = confluenceRestClient.getAllContent(cql, startAt); + List contentList = new ArrayList<>(searchContentItems.getResults()); + total = searchContentItems.getSize(); + startAt += searchContentItems.getResults().size(); + addItemsToQueue(contentList, itemInfoQueue); } while (startAt < total); searchResultsFoundCounter.increment(total); log.info("Number of tickets found in search api call: {}", total); @@ -114,12 +114,12 @@ private void searchForNewContentAndAddToQueue(ConfluenceSourceConfig configurati /** * Add items to queue. * - * @param issueList Issue list. + * @param contentList Content list. * @param itemInfoQueue Item info queue. */ - private void addItemsToQueue(List issueList, Queue itemInfoQueue) { - issueList.forEach(issue -> itemInfoQueue.add(ConfluenceItemInfo.builder() - .withEventTime(Instant.now()).withIssueBean(issue).build())); + private void addItemsToQueue(List contentList, Queue itemInfoQueue) { + contentList.forEach(contentItem -> itemInfoQueue.add(ConfluenceItemInfo.builder() + .withEventTime(Instant.now()).withContentBean(contentItem).build())); } @@ -160,7 +160,7 @@ private StringBuilder createContentFilterCriteria(ConfluenceSourceConfig configu .append(CLOSING_ROUND_BRACKET); } - log.error("Created issue filter criteria JiraQl query: {}", cQl); + log.info("Created content filter criteria ConfluenceQl query: {}", cQl); return cQl; } @@ -170,25 +170,25 @@ private StringBuilder createContentFilterCriteria(ConfluenceSourceConfig configu * @param configuration Input Parameter */ private void validateSpaceFilters(ConfluenceSourceConfig configuration) { - log.trace("Validating project filters"); + log.trace("Validating space filters"); List badFilters = new ArrayList<>(); - Set includedProjects = new HashSet<>(); + Set includedSpaces = new HashSet<>(); List includedAndExcludedSpaces = new ArrayList<>(); Pattern regex = Pattern.compile("[^A-Z0-9]"); - ConfluenceConfigHelper.getSpacesNameIncludeFilter(configuration).forEach(projectFilter -> { - Matcher matcher = regex.matcher(projectFilter); - includedProjects.add(projectFilter); - if (matcher.find() || projectFilter.length() <= 1 || projectFilter.length() > 10) { - badFilters.add(projectFilter); + ConfluenceConfigHelper.getSpacesNameIncludeFilter(configuration).forEach(spaceFilter -> { + Matcher matcher = regex.matcher(spaceFilter); + includedSpaces.add(spaceFilter); + if (matcher.find() || spaceFilter.length() <= 1 || spaceFilter.length() > 10) { + badFilters.add(spaceFilter); } }); - ConfluenceConfigHelper.getSpacesNameExcludeFilter(configuration).forEach(projectFilter -> { - Matcher matcher = regex.matcher(projectFilter); - if (includedProjects.contains(projectFilter)) { - includedAndExcludedSpaces.add(projectFilter); + ConfluenceConfigHelper.getSpacesNameExcludeFilter(configuration).forEach(spaceFilter -> { + Matcher matcher = regex.matcher(spaceFilter); + if (includedSpaces.contains(spaceFilter)) { + includedAndExcludedSpaces.add(spaceFilter); } - if (matcher.find() || projectFilter.length() <= 1 || projectFilter.length() > 10) { - badFilters.add(projectFilter); + if (matcher.find() || spaceFilter.length() <= 1 || spaceFilter.length() > 10) { + badFilters.add(spaceFilter); } }); if (!badFilters.isEmpty()) { diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSource.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSource.java index 69294be228..d126b0da10 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSource.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSource.java @@ -20,7 +20,8 @@ import org.opensearch.dataprepper.model.plugin.PluginFactory; import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.model.source.Source; -import org.opensearch.dataprepper.plugins.source.confluence.rest.auth.ConfluenceAuthConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.rest.auth.AtlassianAuthConfig; import org.opensearch.dataprepper.plugins.source.confluence.utils.ConfluenceConfigHelper; import org.opensearch.dataprepper.plugins.source.source_crawler.CrawlerApplicationContextMarker; import org.opensearch.dataprepper.plugins.source.source_crawler.base.Crawler; @@ -33,37 +34,37 @@ /** - * JiraConnector connector entry point. + * Confluence connector entry point. */ @DataPrepperPlugin(name = PLUGIN_NAME, pluginType = Source.class, pluginConfigurationType = ConfluenceSourceConfig.class, - packagesToScan = {CrawlerApplicationContextMarker.class, ConfluenceSource.class} + packagesToScan = {CrawlerApplicationContextMarker.class, AtlassianSourceConfig.class, ConfluenceSource.class} ) public class ConfluenceSource extends CrawlerSourcePlugin { private static final Logger log = LoggerFactory.getLogger(ConfluenceSource.class); private final ConfluenceSourceConfig confluenceSourceConfig; - private final ConfluenceAuthConfig jiraOauthConfig; + private final AtlassianAuthConfig jiraOauthConfig; @DataPrepperPluginConstructor public ConfluenceSource(final PluginMetrics pluginMetrics, final ConfluenceSourceConfig confluenceSourceConfig, - final ConfluenceAuthConfig jiraOauthConfig, + final AtlassianAuthConfig jiraOauthConfig, final PluginFactory pluginFactory, final AcknowledgementSetManager acknowledgementSetManager, Crawler crawler, PluginExecutorServiceProvider executorServiceProvider) { super(PLUGIN_NAME, pluginMetrics, confluenceSourceConfig, pluginFactory, acknowledgementSetManager, crawler, executorServiceProvider); - log.info("Creating Jira Source Plugin"); + log.info("Creating Confluence Source Plugin"); this.confluenceSourceConfig = confluenceSourceConfig; this.jiraOauthConfig = jiraOauthConfig; } @Override public void start(Buffer> buffer) { - log.info("Starting Jira Source Plugin... "); + log.info("Starting Confluence Source Plugin... "); ConfluenceConfigHelper.validateConfig(confluenceSourceConfig); jiraOauthConfig.initCredentials(); super.start(buffer); @@ -71,7 +72,7 @@ public void start(Buffer> buffer) { @Override public void stop() { - log.info("Stopping Jira Source Plugin"); + log.info("Stopping Confluence Source Plugin"); super.stop(); } diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceConfig.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceConfig.java index 57afe256d2..b22ad46721 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceConfig.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceConfig.java @@ -11,39 +11,13 @@ package org.opensearch.dataprepper.plugins.source.confluence; import com.fasterxml.jackson.annotation.JsonProperty; -import jakarta.validation.Valid; import lombok.Getter; -import org.opensearch.dataprepper.plugins.source.confluence.configuration.AuthenticationConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig; import org.opensearch.dataprepper.plugins.source.confluence.configuration.FilterConfig; import org.opensearch.dataprepper.plugins.source.source_crawler.base.CrawlerSourceConfig; -import java.util.List; - @Getter -public class ConfluenceSourceConfig implements CrawlerSourceConfig { - - private static final int DEFAULT_BATCH_SIZE = 50; - - /** - * Jira account url - */ - @JsonProperty("hosts") - private List hosts; - - /** - * Authentication Config to Access Jira - */ - @JsonProperty("authentication") - @Valid - private AuthenticationConfig authenticationConfig; - - /** - * Batch size for fetching tickets - */ - @JsonProperty("batch_size") - private int batchSize = DEFAULT_BATCH_SIZE; - - +public class ConfluenceSourceConfig extends AtlassianSourceConfig implements CrawlerSourceConfig { /** * Filter Config to filter what tickets get ingested */ @@ -57,11 +31,4 @@ public class ConfluenceSourceConfig implements CrawlerSourceConfig { @JsonProperty("acknowledgments") private boolean acknowledgments = false; - public String getAccountUrl() { - return this.getHosts().get(0); - } - - public String getAuthType() { - return this.getAuthenticationConfig().getAuthType(); - } } diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/AuthenticationConfig.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/AuthenticationConfig.java deleted file mode 100644 index d5632f5f2e..0000000000 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/AuthenticationConfig.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - */ - -package org.opensearch.dataprepper.plugins.source.confluence.configuration; - -import com.fasterxml.jackson.annotation.JsonProperty; -import jakarta.validation.Valid; -import jakarta.validation.constraints.AssertTrue; -import lombok.Getter; - -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.BASIC; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.OAUTH2; - -@Getter -public class AuthenticationConfig { - @JsonProperty("basic") - @Valid - private BasicConfig basicConfig; - - @JsonProperty("oauth2") - @Valid - private Oauth2Config oauth2Config; - - @AssertTrue(message = "Authentication config should have either basic or oauth2") - private boolean isValidAuthenticationConfig() { - boolean hasBasic = basicConfig != null; - boolean hasOauth = oauth2Config != null; - return hasBasic ^ hasOauth; - } - - public String getAuthType() { - if (basicConfig != null) { - return BASIC; - } else { - return OAUTH2; - } - } -} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/BasicConfig.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/BasicConfig.java deleted file mode 100644 index 1cbb6de933..0000000000 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/BasicConfig.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - */ - -package org.opensearch.dataprepper.plugins.source.confluence.configuration; - -import com.fasterxml.jackson.annotation.JsonProperty; -import jakarta.validation.constraints.AssertTrue; -import lombok.Getter; - -@Getter -public class BasicConfig { - @JsonProperty("username") - private String username; - - @JsonProperty("password") - private String password; - - @AssertTrue(message = "Username and Password are both required for Basic Auth") - private boolean isBasicConfigValid() { - return username != null && password != null; - } -} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/Oauth2Config.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/Oauth2Config.java deleted file mode 100644 index 22daa7841b..0000000000 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/Oauth2Config.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - */ - -package org.opensearch.dataprepper.plugins.source.confluence.configuration; - -import com.fasterxml.jackson.annotation.JsonProperty; -import jakarta.validation.constraints.AssertTrue; -import lombok.Getter; -import org.opensearch.dataprepper.model.plugin.PluginConfigVariable; - -@Getter -public class Oauth2Config { - @JsonProperty("client_id") - private String clientId; - - @JsonProperty("client_secret") - private String clientSecret; - - @JsonProperty("access_token") - private PluginConfigVariable accessToken; - - @JsonProperty("refresh_token") - private PluginConfigVariable refreshToken; - - @AssertTrue(message = "Client ID, Client Secret, Access Token, and Refresh Token are both required for Oauth2") - private boolean isOauth2ConfigValid() { - return clientId != null && clientSecret != null && accessToken != null && refreshToken != null; - } -} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/BasicAuthInterceptor.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/BasicAuthInterceptor.java deleted file mode 100644 index 60bada4963..0000000000 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/BasicAuthInterceptor.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - */ - -package org.opensearch.dataprepper.plugins.source.confluence.rest; - -import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceSourceConfig; -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpRequest; -import org.springframework.http.client.ClientHttpRequestExecution; -import org.springframework.http.client.ClientHttpRequestInterceptor; -import org.springframework.http.client.ClientHttpResponse; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.Base64; - - -public class BasicAuthInterceptor implements ClientHttpRequestInterceptor { - private final String username; - private final String password; - - public BasicAuthInterceptor(ConfluenceSourceConfig config) { - this.username = config.getAuthenticationConfig().getBasicConfig().getUsername(); - this.password = config.getAuthenticationConfig().getBasicConfig().getPassword(); - } - - @Override - public ClientHttpResponse intercept(HttpRequest request, byte[] body, ClientHttpRequestExecution execution) throws IOException { - String auth = username + ":" + password; - byte[] encodedAuth = Base64.getEncoder().encode(auth.getBytes(StandardCharsets.US_ASCII)); - String authHeader = "Basic " + new String(encodedAuth); - request.getHeaders().set(HttpHeaders.AUTHORIZATION, authHeader); - return execution.execute(request, body); - } -} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClient.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClient.java index 45087d153f..bcda06d076 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClient.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClient.java @@ -16,30 +16,22 @@ import io.micrometer.core.instrument.Timer; import lombok.extern.slf4j.Slf4j; import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.plugins.source.atlassian.rest.AtlassianRestClient; +import org.opensearch.dataprepper.plugins.source.atlassian.rest.auth.AtlassianAuthConfig; import org.opensearch.dataprepper.plugins.source.confluence.models.ConfluenceSearchResults; -import org.opensearch.dataprepper.plugins.source.confluence.rest.auth.ConfluenceAuthConfig; -import org.opensearch.dataprepper.plugins.source.source_crawler.exception.BadRequestException; -import org.opensearch.dataprepper.plugins.source.source_crawler.exception.UnAuthorizedException; -import org.opensearch.dataprepper.plugins.source.source_crawler.utils.AddressValidation; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; -import org.springframework.web.client.HttpClientErrorException; import org.springframework.web.client.RestTemplate; import org.springframework.web.util.UriComponentsBuilder; import javax.inject.Named; import java.net.URI; -import java.util.List; -import static org.opensearch.dataprepper.logging.DataPrepperMarkers.NOISY; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.RETRY_ATTEMPT; import static org.opensearch.dataprepper.plugins.source.confluence.utils.CqlConstants.CQL_FIELD; import static org.opensearch.dataprepper.plugins.source.confluence.utils.CqlConstants.EXPAND_FIELD; import static org.opensearch.dataprepper.plugins.source.confluence.utils.CqlConstants.EXPAND_VALUE; @Slf4j @Named -public class ConfluenceRestClient { +public class ConfluenceRestClient extends AtlassianRestClient { public static final String REST_API_SEARCH = "wiki/rest/api/content/search"; public static final String REST_API_FETCH_CONTENT = "wiki/rest/api/content/"; @@ -48,21 +40,21 @@ public class ConfluenceRestClient { public static final String FIFTY = "50"; public static final String START_AT = "startAt"; public static final String MAX_RESULT = "maxResults"; - public static final List RETRY_ATTEMPT_SLEEP_TIME = List.of(1, 2, 5, 10, 20, 40); private static final String PAGE_FETCH_LATENCY_TIMER = "pageFetchLatency"; private static final String SEARCH_CALL_LATENCY_TIMER = "searchCallLatency"; private static final String SPACES_FETCH_LATENCY_TIMER = "spacesFetchLatency"; private static final String PAGES_REQUESTED = "pagesRequested"; private int sleepTimeMultiplier = 1000; private final RestTemplate restTemplate; - private final ConfluenceAuthConfig authConfig; + private final AtlassianAuthConfig authConfig; private final Timer contentFetchLatencyTimer; private final Timer searchCallLatencyTimer; private final Timer spaceFetchLatencyTimer; private final Counter contentRequestedCounter; - public ConfluenceRestClient(RestTemplate restTemplate, ConfluenceAuthConfig authConfig, + public ConfluenceRestClient(RestTemplate restTemplate, AtlassianAuthConfig authConfig, PluginMetrics pluginMetrics) { + super(restTemplate, authConfig); this.restTemplate = restTemplate; this.authConfig = authConfig; @@ -107,36 +99,6 @@ public String getContent(String contentId) { return invokeRestApi(uri, String.class).getBody(); } - private ResponseEntity invokeRestApi(URI uri, Class responseType) throws BadRequestException { - AddressValidation.validateInetAddress(AddressValidation.getInetAddress(uri.toString())); - int retryCount = 0; - while (retryCount < RETRY_ATTEMPT) { - try { - return restTemplate.getForEntity(uri, responseType); - } catch (HttpClientErrorException ex) { - HttpStatus statusCode = ex.getStatusCode(); - String statusMessage = ex.getMessage(); - log.error("An exception has occurred while getting response from Jira search API {}", ex.getMessage()); - if (statusCode == HttpStatus.FORBIDDEN) { - throw new UnAuthorizedException(statusMessage); - } else if (statusCode == HttpStatus.UNAUTHORIZED) { - log.error(NOISY, "Token expired. We will try to renew the tokens now", ex); - authConfig.renewCredentials(); - } else if (statusCode == HttpStatus.TOO_MANY_REQUESTS) { - log.error(NOISY, "Hitting API rate limit. Backing off with sleep timer.", ex); - } - try { - Thread.sleep((long) RETRY_ATTEMPT_SLEEP_TIME.get(retryCount) * sleepTimeMultiplier); - } catch (InterruptedException e) { - throw new RuntimeException("Sleep in the retry attempt got interrupted", e); - } - } - retryCount++; - } - String errorMessage = String.format("Exceeded max retry attempts. Failed to execute the Rest API call %s", uri); - log.error(errorMessage); - throw new RuntimeException(errorMessage); - } @VisibleForTesting public void setSleepTimeMultiplier(int multiplier) { diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/CustomRestTemplateConfig.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/CustomRestTemplateConfig.java deleted file mode 100644 index 197ede2ac2..0000000000 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/CustomRestTemplateConfig.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - */ - -package org.opensearch.dataprepper.plugins.source.confluence.rest; - - -import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceSourceConfig; -import org.opensearch.dataprepper.plugins.source.confluence.rest.auth.ConfluenceAuthConfig; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.http.client.ClientHttpRequestInterceptor; -import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; -import org.springframework.web.client.RestTemplate; - -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.OAUTH2; - -@Configuration -public class CustomRestTemplateConfig { - - @Bean - public RestTemplate basicAuthRestTemplate(ConfluenceSourceConfig config, ConfluenceAuthConfig authConfig) { - RestTemplate restTemplate = new RestTemplate(); - restTemplate.setRequestFactory(new HttpComponentsClientHttpRequestFactory()); - ClientHttpRequestInterceptor httpInterceptor; - if (OAUTH2.equals(config.getAuthType())) { - httpInterceptor = new OAuth2RequestInterceptor(authConfig); - } else { - httpInterceptor = new BasicAuthInterceptor(config); - } - restTemplate.getInterceptors().add(httpInterceptor); - return restTemplate; - } - - -} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/OAuth2RequestInterceptor.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/OAuth2RequestInterceptor.java deleted file mode 100644 index 5488fd0850..0000000000 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/OAuth2RequestInterceptor.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - */ - -package org.opensearch.dataprepper.plugins.source.confluence.rest; - -import org.opensearch.dataprepper.plugins.source.confluence.rest.auth.ConfluenceAuthConfig; -import org.opensearch.dataprepper.plugins.source.confluence.rest.auth.ConfluenceOauthConfig; -import org.springframework.http.HttpRequest; -import org.springframework.http.client.ClientHttpRequestExecution; -import org.springframework.http.client.ClientHttpRequestInterceptor; -import org.springframework.http.client.ClientHttpResponse; - -import java.io.IOException; - -public class OAuth2RequestInterceptor implements ClientHttpRequestInterceptor { - - private final ConfluenceAuthConfig config; - - public OAuth2RequestInterceptor(ConfluenceAuthConfig config) { - this.config = config; - } - - @Override - public ClientHttpResponse intercept(HttpRequest request, byte[] body, ClientHttpRequestExecution execution) throws IOException { - request.getHeaders().setBearerAuth(((ConfluenceOauthConfig) config).getAccessToken()); - return execution.execute(request, body); - } - -} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceAuthConfig.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceAuthConfig.java deleted file mode 100644 index 4104601261..0000000000 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceAuthConfig.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - */ - -package org.opensearch.dataprepper.plugins.source.confluence.rest.auth; - -/** - * The interface that defines the behaviour for Jira auth configs. - */ -public interface ConfluenceAuthConfig { - - /** - * Returns the URL for the Jira instance. - * - * @return the URL for the Jira instance. - */ - String getUrl(); - - /** - * Initializes the credentials for the Jira instance. - */ - void initCredentials(); - - /** - * Renews the credentials for the Jira instance. - */ - void renewCredentials(); -} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceAuthFactory.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceAuthFactory.java deleted file mode 100644 index 08b4e929cb..0000000000 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceAuthFactory.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - */ - -package org.opensearch.dataprepper.plugins.source.confluence.rest.auth; - -import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceSourceConfig; -import org.springframework.beans.factory.FactoryBean; -import org.springframework.context.annotation.Configuration; - -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.OAUTH2; - -@Configuration -public class ConfluenceAuthFactory implements FactoryBean { - - private final ConfluenceSourceConfig sourceConfig; - - public ConfluenceAuthFactory(ConfluenceSourceConfig sourceConfig) { - this.sourceConfig = sourceConfig; - } - - @Override - public ConfluenceAuthConfig getObject() { - String authType = sourceConfig.getAuthType(); - if (OAUTH2.equals(authType)) { - return new ConfluenceOauthConfig(sourceConfig); - } - return new ConfluenceBasicAuthConfig(sourceConfig); - } - - @Override - public Class getObjectType() { - return ConfluenceAuthConfig.class; - } -} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceBasicAuthConfig.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceBasicAuthConfig.java deleted file mode 100644 index e400dbf7b0..0000000000 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceBasicAuthConfig.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - */ - -package org.opensearch.dataprepper.plugins.source.confluence.rest.auth; - - -import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceSourceConfig; - -public class ConfluenceBasicAuthConfig implements ConfluenceAuthConfig { - - private String accountUrl; - private final ConfluenceSourceConfig confluenceSourceConfig; - - public ConfluenceBasicAuthConfig(ConfluenceSourceConfig confluenceSourceConfig) { - this.confluenceSourceConfig = confluenceSourceConfig; - accountUrl = confluenceSourceConfig.getAccountUrl(); - if (!accountUrl.endsWith("/")) { - accountUrl += "/"; - } - } - - @Override - public String getUrl() { - return accountUrl; - } - - @Override - public void initCredentials() { - //do nothing for basic authentication - } - - @Override - public void renewCredentials() { - //do nothing for basic authentication - } - - -} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceOauthConfig.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceOauthConfig.java deleted file mode 100644 index 2c1fb08afb..0000000000 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceOauthConfig.java +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - */ - -package org.opensearch.dataprepper.plugins.source.confluence.rest.auth; - -import lombok.Getter; -import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceSourceConfig; -import org.opensearch.dataprepper.plugins.source.confluence.configuration.Oauth2Config; -import org.opensearch.dataprepper.plugins.source.source_crawler.exception.UnAuthorizedException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.http.HttpEntity; -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpMethod; -import org.springframework.http.HttpStatus; -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; -import org.springframework.util.StringUtils; -import org.springframework.web.client.HttpClientErrorException; -import org.springframework.web.client.RestTemplate; - -import java.time.Instant; -import java.util.List; -import java.util.Map; - -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.RETRY_ATTEMPT; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.CqlConstants.SLASH; - -/** - * The type Jira service. - */ - -public class ConfluenceOauthConfig implements ConfluenceAuthConfig { - - public static final String OAuth2_URL = "https://api.atlassian.com/ex/jira/"; - public static final String ACCESSIBLE_RESOURCES = "https://api.atlassian.com/oauth/token/accessible-resources"; - public static final String TOKEN_LOCATION = "https://auth.atlassian.com/oauth/token"; - - public static final String EXPIRES_IN = "expires_in"; - public static final String REFRESH_TOKEN = "refresh_token"; - public static final String ACCESS_TOKEN = "access_token"; - private static final Logger log = LoggerFactory.getLogger(ConfluenceOauthConfig.class); - RestTemplate restTemplate = new RestTemplate(); - private String url; - @Getter - private int expiresInSeconds = 0; - @Getter - private Instant expireTime = Instant.ofEpochMilli(0); - @Getter - private String accessToken; - @Getter - private String refreshToken; - private String cloudId = null; - private final String clientId; - private final String clientSecret; - private final ConfluenceSourceConfig confluenceSourceConfig; - private final Object cloudIdFetchLock = new Object(); - private final Object tokenRenewLock = new Object(); - - public ConfluenceOauthConfig(ConfluenceSourceConfig confluenceSourceConfig) { - this.confluenceSourceConfig = confluenceSourceConfig; - this.accessToken = (String) confluenceSourceConfig.getAuthenticationConfig().getOauth2Config() - .getAccessToken().getValue(); - this.refreshToken = (String) confluenceSourceConfig.getAuthenticationConfig() - .getOauth2Config().getRefreshToken().getValue(); - this.clientId = confluenceSourceConfig.getAuthenticationConfig().getOauth2Config().getClientId(); - this.clientSecret = confluenceSourceConfig.getAuthenticationConfig().getOauth2Config().getClientSecret(); - } - - public String getJiraAccountCloudId() { - log.info("Getting Jira Account Cloud ID"); - synchronized (cloudIdFetchLock) { - if (this.cloudId != null) { - //Someone else must have initialized it - return this.cloudId; - } - - int retryCount = 0; - while (retryCount < RETRY_ATTEMPT) { - retryCount++; - try { - HttpHeaders headers = new HttpHeaders(); - headers.setBearerAuth(accessToken); - headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED); - HttpEntity entity = new HttpEntity<>(headers); - ResponseEntity exchangeResponse = - restTemplate.exchange(ACCESSIBLE_RESOURCES, HttpMethod.GET, entity, Object.class); - List> listResponse = (List>) exchangeResponse.getBody(); - Map response = listResponse.get(0); - this.cloudId = (String) response.get("id"); - return this.cloudId; - } catch (HttpClientErrorException e) { - if (e.getRawStatusCode() == HttpStatus.UNAUTHORIZED.value()) { - renewCredentials(); - } - log.error("Error occurred while accessing resources: ", e); - } - } - throw new UnAuthorizedException(String.format("Access token expired. Unable to renew even after %s attempts", RETRY_ATTEMPT)); - } - } - - public void renewCredentials() { - Instant currentTime = Instant.now(); - if (expireTime.isAfter(currentTime)) { - //There is still time to renew or someone else must have already renewed it - return; - } - - synchronized (tokenRenewLock) { - if (expireTime.isAfter(currentTime)) { - //Someone else must have already renewed it - return; - } - - log.info("Renewing access token and refresh token pair for Jira Connector."); - HttpHeaders headers = new HttpHeaders(); - headers.setContentType(MediaType.APPLICATION_JSON); - String payloadTemplate = "{\"grant_type\": \"%s\", \"client_id\": \"%s\", \"client_secret\": \"%s\", \"refresh_token\": \"%s\"}"; - String payload = String.format(payloadTemplate, "refresh_token", clientId, clientSecret, refreshToken); - HttpEntity entity = new HttpEntity<>(payload, headers); - - Oauth2Config oauth2Config = confluenceSourceConfig.getAuthenticationConfig().getOauth2Config(); - try { - ResponseEntity responseEntity = restTemplate.postForEntity(TOKEN_LOCATION, entity, Map.class); - Map oauthClientResponse = responseEntity.getBody(); - this.accessToken = (String) oauthClientResponse.get(ACCESS_TOKEN); - this.refreshToken = (String) oauthClientResponse.get(REFRESH_TOKEN); - this.expiresInSeconds = (int) oauthClientResponse.get(EXPIRES_IN); - this.expireTime = Instant.now().plusSeconds(expiresInSeconds); - // updating config object's PluginConfigVariable so that it updates the underlying Secret store - oauth2Config.getAccessToken().setValue(this.accessToken); - oauth2Config.getRefreshToken().setValue(this.refreshToken); - log.info("Access Token and Refresh Token pair is now refreshed. Corresponding Secret store key updated."); - } catch (HttpClientErrorException ex) { - this.expireTime = Instant.ofEpochMilli(0); - this.expiresInSeconds = 0; - HttpStatus statusCode = ex.getStatusCode(); - log.error("Failed to renew access token. Status code: {}, Error Message: {}", - statusCode, ex.getMessage()); - if (statusCode == HttpStatus.FORBIDDEN || statusCode == HttpStatus.UNAUTHORIZED) { - log.info("Trying to refresh the secrets"); - // Refreshing the secrets. It should help if someone already renewed the tokens. - // Refreshing one of the secret refreshes the entire store so triggering refresh on just one - oauth2Config.getAccessToken().refresh(); - this.accessToken = (String) oauth2Config.getAccessToken().getValue(); - this.refreshToken = (String) oauth2Config.getRefreshToken().getValue(); - this.expireTime = Instant.now().plusSeconds(10); - } - throw new RuntimeException("Failed to renew access token message:" + ex.getMessage(), ex); - } - } - } - - @Override - public String getUrl() { - if (!StringUtils.hasLength(url)) { - synchronized (cloudIdFetchLock) { - if (!StringUtils.hasLength(url)) { - initCredentials(); - } - } - } - return url; - } - - /** - * Method for getting Jira url based on auth type. - */ - @Override - public void initCredentials() { - //For OAuth based flow, we use a different Jira url - this.cloudId = getJiraAccountCloudId(); - this.url = OAuth2_URL + this.cloudId + SLASH; - } -} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/Constants.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/Constants.java index 114c33cebd..e63d001a87 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/Constants.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/Constants.java @@ -27,6 +27,5 @@ public class Constants { public static final String CONTENT_ID = "id"; public static final String CREATED = "created"; public static final String BASIC = "Basic"; - public static final String LIVE = "live"; public static final String PLUGIN_NAME = "confluence"; } \ No newline at end of file diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClientTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClientTest.java index 27a0885208..ea7f35350e 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClientTest.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClientTest.java @@ -20,13 +20,13 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.plugins.source.atlassian.rest.auth.AtlassianAuthConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.rest.auth.AtlassianAuthFactory; import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceServiceTest; import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceSourceConfig; -import org.opensearch.dataprepper.plugins.source.confluence.exception.BadRequestException; -import org.opensearch.dataprepper.plugins.source.confluence.exception.UnAuthorizedException; import org.opensearch.dataprepper.plugins.source.confluence.models.ConfluenceSearchResults; -import org.opensearch.dataprepper.plugins.source.confluence.rest.auth.ConfluenceAuthConfig; -import org.opensearch.dataprepper.plugins.source.confluence.rest.auth.ConfluenceAuthFactory; +import org.opensearch.dataprepper.plugins.source.source_crawler.exception.BadRequestException; +import org.opensearch.dataprepper.plugins.source.source_crawler.exception.UnAuthorizedException; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.client.HttpClientErrorException; @@ -57,7 +57,7 @@ public class ConfluenceRestClientTest { private RestTemplate restTemplate; @Mock - private ConfluenceAuthConfig authConfig; + private AtlassianAuthConfig authConfig; private final PluginMetrics pluginMetrics = PluginMetrics.fromNames("jiraRestClient", "aws"); @@ -76,7 +76,7 @@ public void testFetchingJiraIssue(String configFileName) { String exampleTicketResponse = "{\"id\":\"123\",\"key\":\"key\",\"self\":\"https://example.com/rest/api/2/issue/123\"}"; doReturn(new ResponseEntity<>(exampleTicketResponse, HttpStatus.OK)).when(restTemplate).getForEntity(any(URI.class), any(Class.class)); ConfluenceSourceConfig confluenceSourceConfig = ConfluenceServiceTest.createJiraConfigurationFromYaml(configFileName); - ConfluenceAuthConfig authConfig = new ConfluenceAuthFactory(confluenceSourceConfig).getObject(); + AtlassianAuthConfig authConfig = new AtlassianAuthFactory(confluenceSourceConfig).getObject(); ConfluenceRestClient confluenceRestClient = new ConfluenceRestClient(restTemplate, authConfig, pluginMetrics); String ticketDetails = confluenceRestClient.getContent("key"); assertEquals(exampleTicketResponse, ticketDetails);