functional with making use of Atlassian Commons package

Signed-off-by: Santhosh Gandhe <[email protected]>
opensearch-project · Feb 3, 2025 · 0e89bf0 · 0e89bf0
1 parent 04ac7c0
commit 0e89bf0
Show file tree

Hide file tree

Showing 24 changed files with 58 additions and 660 deletions.
diff --git a/.../configuration/AtlassianSourceConfig.java → ...urce/atlassian/AtlassianSourceConfig.java b/.../configuration/AtlassianSourceConfig.java → ...urce/atlassian/AtlassianSourceConfig.java
@@ -8,11 +8,12 @@
  *
  */
 
-package org.opensearch.dataprepper.plugins.source.atlassian.configuration;
+package org.opensearch.dataprepper.plugins.source.atlassian;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import jakarta.validation.Valid;
 import lombok.Getter;
+import org.opensearch.dataprepper.plugins.source.atlassian.configuration.AuthenticationConfig;
 import org.opensearch.dataprepper.plugins.source.source_crawler.base.CrawlerSourceConfig;
 
 import java.util.List;

diff --git a/...n/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/BasicAuthInterceptor.java b/...n/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/BasicAuthInterceptor.java
@@ -10,7 +10,7 @@
 
 package org.opensearch.dataprepper.plugins.source.atlassian.rest;
 
-import org.opensearch.dataprepper.plugins.source.atlassian.configuration.AtlassianSourceConfig;
+import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig;
 import org.springframework.http.HttpHeaders;
 import org.springframework.http.HttpRequest;
 import org.springframework.http.client.ClientHttpRequestExecution;

diff --git a/...va/org/opensearch/dataprepper/plugins/source/atlassian/rest/CustomRestTemplateConfig.java b/...va/org/opensearch/dataprepper/plugins/source/atlassian/rest/CustomRestTemplateConfig.java
@@ -11,7 +11,7 @@
 package org.opensearch.dataprepper.plugins.source.atlassian.rest;
 
 
-import org.opensearch.dataprepper.plugins.source.atlassian.configuration.AtlassianSourceConfig;
+import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig;
 import org.opensearch.dataprepper.plugins.source.atlassian.rest.auth.AtlassianAuthConfig;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;

diff --git a/...a/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianAuthFactory.java b/...a/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianAuthFactory.java
@@ -10,7 +10,7 @@
 
 package org.opensearch.dataprepper.plugins.source.atlassian.rest.auth;
 
-import org.opensearch.dataprepper.plugins.source.atlassian.configuration.AtlassianSourceConfig;
+import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig;
 import org.springframework.beans.factory.FactoryBean;
 import org.springframework.context.annotation.Configuration;
 

diff --git a/...g/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianBasicAuthConfig.java b/...g/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianBasicAuthConfig.java
@@ -11,7 +11,7 @@
 package org.opensearch.dataprepper.plugins.source.atlassian.rest.auth;
 
 
-import org.opensearch.dataprepper.plugins.source.atlassian.configuration.AtlassianSourceConfig;
+import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig;
 
 public class AtlassianBasicAuthConfig implements AtlassianAuthConfig {
 

diff --git a/...a/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianOauthConfig.java b/...a/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianOauthConfig.java
@@ -11,7 +11,7 @@
 package org.opensearch.dataprepper.plugins.source.atlassian.rest.auth;
 
 import lombok.Getter;
-import org.opensearch.dataprepper.plugins.source.atlassian.configuration.AtlassianSourceConfig;
+import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig;
 import org.opensearch.dataprepper.plugins.source.atlassian.configuration.Oauth2Config;
 import org.opensearch.dataprepper.plugins.source.source_crawler.exception.UnAuthorizedException;
 import org.slf4j.Logger;

diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/build.gradle b/data-prepper-plugins/saas-source-plugins/confluence-source/build.gradle
@@ -5,6 +5,7 @@ plugins {
 dependencies {
 
     implementation project(path: ':data-prepper-plugins:saas-source-plugins:source-crawler')
+    implementation project(path: ':data-prepper-plugins:saas-source-plugins:atlassian-commons')
     implementation project(path: ':data-prepper-api')
     implementation project(path: ':data-prepper-plugins:aws-plugin-api')
     implementation project(path: ':data-prepper-plugins:buffer-common')

diff --git a/...rc/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceItemInfo.java b/...rc/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceItemInfo.java
@@ -137,7 +137,7 @@ public ConfluenceItemInfoBuilder withSpace(String space) {
             return this;
         }
 
-        public ConfluenceItemInfoBuilder withIssueBean(ConfluenceItem contentItem) {
+        public ConfluenceItemInfoBuilder withContentBean(ConfluenceItem contentItem) {
             Map<String, Object> contentItemMetadata = new HashMap<>();
             contentItemMetadata.put(SPACE_KEY, contentItem.getSpaceItem().getKey());
             contentItemMetadata.put(CONTENT_TITLE, contentItem.getTitle());

diff --git a/...src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java b/...src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java
@@ -48,7 +48,7 @@
 
 
 /**
- * Service class for interactive external Atlassian jira SaaS service and fetch required details using their rest apis.
+ * Service class for interactive external Atlassian Confluence SaaS service and fetch required details using their rest apis.
  */
 
 @Slf4j
@@ -73,7 +73,7 @@ public ConfluenceService(ConfluenceSourceConfig confluenceSourceConfig,
     }
 
     /**
-     * Get jira entities.
+     * Get Confluence entities.
      *
      * @param configuration the configuration.
      * @param timestamp     timestamp.
@@ -89,7 +89,7 @@ public String getContent(String contentId) {
     }
 
     /**
-     * Method for building Issue Item Info.
+     * Method for building Content Item Info.
      *
      * @param configuration Input Parameter
      * @param timestamp     Input Parameter
@@ -101,11 +101,11 @@ private void searchForNewContentAndAddToQueue(ConfluenceSourceConfig configurati
         int total;
         int startAt = 0;
         do {
-            ConfluenceSearchResults searchIssues = confluenceRestClient.getAllContent(cql, startAt);
-            List<ConfluenceItem> issueList = new ArrayList<>(searchIssues.getResults());
-            total = searchIssues.getSize();
-            startAt += searchIssues.getResults().size();
-            addItemsToQueue(issueList, itemInfoQueue);
+            ConfluenceSearchResults searchContentItems = confluenceRestClient.getAllContent(cql, startAt);
+            List<ConfluenceItem> contentList = new ArrayList<>(searchContentItems.getResults());
+            total = searchContentItems.getSize();
+            startAt += searchContentItems.getResults().size();
+            addItemsToQueue(contentList, itemInfoQueue);
         } while (startAt < total);
         searchResultsFoundCounter.increment(total);
         log.info("Number of tickets found in search api call: {}", total);
@@ -114,12 +114,12 @@ private void searchForNewContentAndAddToQueue(ConfluenceSourceConfig configurati
     /**
      * Add items to queue.
      *
-     * @param issueList     Issue list.
+     * @param contentList   Content list.
      * @param itemInfoQueue Item info queue.
      */
-    private void addItemsToQueue(List<ConfluenceItem> issueList, Queue<ItemInfo> itemInfoQueue) {
-        issueList.forEach(issue -> itemInfoQueue.add(ConfluenceItemInfo.builder()
-                .withEventTime(Instant.now()).withIssueBean(issue).build()));
+    private void addItemsToQueue(List<ConfluenceItem> contentList, Queue<ItemInfo> itemInfoQueue) {
+        contentList.forEach(contentItem -> itemInfoQueue.add(ConfluenceItemInfo.builder()
+                .withEventTime(Instant.now()).withContentBean(contentItem).build()));
     }
 
 
@@ -160,7 +160,7 @@ private StringBuilder createContentFilterCriteria(ConfluenceSourceConfig configu
                     .append(CLOSING_ROUND_BRACKET);
         }
 
-        log.error("Created issue filter criteria JiraQl query: {}", cQl);
+        log.info("Created content filter criteria ConfluenceQl query: {}", cQl);
         return cQl;
     }
 
@@ -170,25 +170,25 @@ private StringBuilder createContentFilterCriteria(ConfluenceSourceConfig configu
      * @param configuration Input Parameter
      */
     private void validateSpaceFilters(ConfluenceSourceConfig configuration) {
-        log.trace("Validating project filters");
+        log.trace("Validating space filters");
         List<String> badFilters = new ArrayList<>();
-        Set<String> includedProjects = new HashSet<>();
+        Set<String> includedSpaces = new HashSet<>();
         List<String> includedAndExcludedSpaces = new ArrayList<>();
         Pattern regex = Pattern.compile("[^A-Z0-9]");
-        ConfluenceConfigHelper.getSpacesNameIncludeFilter(configuration).forEach(projectFilter -> {
-            Matcher matcher = regex.matcher(projectFilter);
-            includedProjects.add(projectFilter);
-            if (matcher.find() || projectFilter.length() <= 1 || projectFilter.length() > 10) {
-                badFilters.add(projectFilter);
+        ConfluenceConfigHelper.getSpacesNameIncludeFilter(configuration).forEach(spaceFilter -> {
+            Matcher matcher = regex.matcher(spaceFilter);
+            includedSpaces.add(spaceFilter);
+            if (matcher.find() || spaceFilter.length() <= 1 || spaceFilter.length() > 10) {
+                badFilters.add(spaceFilter);
             }
         });
-        ConfluenceConfigHelper.getSpacesNameExcludeFilter(configuration).forEach(projectFilter -> {
-            Matcher matcher = regex.matcher(projectFilter);
-            if (includedProjects.contains(projectFilter)) {
-                includedAndExcludedSpaces.add(projectFilter);
+        ConfluenceConfigHelper.getSpacesNameExcludeFilter(configuration).forEach(spaceFilter -> {
+            Matcher matcher = regex.matcher(spaceFilter);
+            if (includedSpaces.contains(spaceFilter)) {
+                includedAndExcludedSpaces.add(spaceFilter);
             }
-            if (matcher.find() || projectFilter.length() <= 1 || projectFilter.length() > 10) {
-                badFilters.add(projectFilter);
+            if (matcher.find() || spaceFilter.length() <= 1 || spaceFilter.length() > 10) {
+                badFilters.add(spaceFilter);
             }
         });
         if (!badFilters.isEmpty()) {

diff --git a/.../src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSource.java b/.../src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSource.java
@@ -20,7 +20,8 @@
 import org.opensearch.dataprepper.model.plugin.PluginFactory;
 import org.opensearch.dataprepper.model.record.Record;
 import org.opensearch.dataprepper.model.source.Source;
-import org.opensearch.dataprepper.plugins.source.confluence.rest.auth.ConfluenceAuthConfig;
+import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig;
+import org.opensearch.dataprepper.plugins.source.atlassian.rest.auth.AtlassianAuthConfig;
 import org.opensearch.dataprepper.plugins.source.confluence.utils.ConfluenceConfigHelper;
 import org.opensearch.dataprepper.plugins.source.source_crawler.CrawlerApplicationContextMarker;
 import org.opensearch.dataprepper.plugins.source.source_crawler.base.Crawler;
@@ -33,45 +34,45 @@
 
 
 /**
- * JiraConnector connector entry point.
+ * Confluence connector entry point.
  */
 
 @DataPrepperPlugin(name = PLUGIN_NAME,
         pluginType = Source.class,
         pluginConfigurationType = ConfluenceSourceConfig.class,
-        packagesToScan = {CrawlerApplicationContextMarker.class, ConfluenceSource.class}
+        packagesToScan = {CrawlerApplicationContextMarker.class, AtlassianSourceConfig.class, ConfluenceSource.class}
 )
 public class ConfluenceSource extends CrawlerSourcePlugin {
 
     private static final Logger log = LoggerFactory.getLogger(ConfluenceSource.class);
     private final ConfluenceSourceConfig confluenceSourceConfig;
-    private final ConfluenceAuthConfig jiraOauthConfig;
+    private final AtlassianAuthConfig jiraOauthConfig;
 
     @DataPrepperPluginConstructor
     public ConfluenceSource(final PluginMetrics pluginMetrics,
                             final ConfluenceSourceConfig confluenceSourceConfig,
-                            final ConfluenceAuthConfig jiraOauthConfig,
+                            final AtlassianAuthConfig jiraOauthConfig,
                             final PluginFactory pluginFactory,
                             final AcknowledgementSetManager acknowledgementSetManager,
                             Crawler crawler,
                             PluginExecutorServiceProvider executorServiceProvider) {
         super(PLUGIN_NAME, pluginMetrics, confluenceSourceConfig, pluginFactory, acknowledgementSetManager, crawler, executorServiceProvider);
-        log.info("Creating Jira Source Plugin");
+        log.info("Creating Confluence Source Plugin");
         this.confluenceSourceConfig = confluenceSourceConfig;
         this.jiraOauthConfig = jiraOauthConfig;
     }
 
     @Override
     public void start(Buffer<Record<Event>> buffer) {
-        log.info("Starting Jira Source Plugin... ");
+        log.info("Starting Confluence Source Plugin... ");
         ConfluenceConfigHelper.validateConfig(confluenceSourceConfig);
         jiraOauthConfig.initCredentials();
         super.start(buffer);
     }
 
     @Override
     public void stop() {
-        log.info("Stopping Jira Source Plugin");
+        log.info("Stopping Confluence Source Plugin");
         super.stop();
     }
 

diff --git a/...ain/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceConfig.java b/...ain/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceConfig.java
@@ -11,39 +11,13 @@
 package org.opensearch.dataprepper.plugins.source.confluence;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
-import jakarta.validation.Valid;
 import lombok.Getter;
-import org.opensearch.dataprepper.plugins.source.confluence.configuration.AuthenticationConfig;
+import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig;
 import org.opensearch.dataprepper.plugins.source.confluence.configuration.FilterConfig;
 import org.opensearch.dataprepper.plugins.source.source_crawler.base.CrawlerSourceConfig;
 
-import java.util.List;
-
 @Getter
-public class ConfluenceSourceConfig implements CrawlerSourceConfig {
-
-    private static final int DEFAULT_BATCH_SIZE = 50;
-
-    /**
-     * Jira account url
-     */
-    @JsonProperty("hosts")
-    private List<String> hosts;
-
-    /**
-     * Authentication Config to Access Jira
-     */
-    @JsonProperty("authentication")
-    @Valid
-    private AuthenticationConfig authenticationConfig;
-
-    /**
-     * Batch size for fetching tickets
-     */
-    @JsonProperty("batch_size")
-    private int batchSize = DEFAULT_BATCH_SIZE;
-
-
+public class ConfluenceSourceConfig extends AtlassianSourceConfig implements CrawlerSourceConfig {
     /**
      * Filter Config to filter what tickets get ingested
      */
@@ -57,11 +31,4 @@ public class ConfluenceSourceConfig implements CrawlerSourceConfig {
     @JsonProperty("acknowledgments")
     private boolean acknowledgments = false;
 
-    public String getAccountUrl() {
-        return this.getHosts().get(0);
-    }
-
-    public String getAuthType() {
-        return this.getAuthenticationConfig().getAuthType();
-    }
 }
diff --git a/.../opensearch/dataprepper/plugins/source/confluence/configuration/AuthenticationConfig.java b/.../opensearch/dataprepper/plugins/source/confluence/configuration/AuthenticationConfig.java
diff --git a/.../java/org/opensearch/dataprepper/plugins/source/confluence/configuration/BasicConfig.java b/.../java/org/opensearch/dataprepper/plugins/source/confluence/configuration/BasicConfig.java