From ae65748f404b1082450c267d567f5642a56341e2 Mon Sep 17 00:00:00 2001 From: Santhosh Gandhe <1909520+san81@users.noreply.github.com> Date: Wed, 29 Jan 2025 16:38:27 -0800 Subject: [PATCH 01/14] confluence source Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com> --- .../confluence-source/README.md | 9 + .../confluence-source/build.gradle | 33 ++ .../source/confluence/ConfluenceClient.java | 146 ++++++++ .../source/confluence/ConfluenceItemInfo.java | 158 +++++++++ .../source/confluence/ConfluenceIterator.java | 124 +++++++ .../source/confluence/ConfluenceService.java | 217 ++++++++++++ .../source/confluence/ConfluenceSource.java | 78 +++++ .../confluence/ConfluenceSourceConfig.java | 67 ++++ .../configuration/AuthenticationConfig.java | 45 +++ .../confluence/configuration/BasicConfig.java | 29 ++ .../configuration/FilterConfig.java | 26 ++ .../configuration/IssueTypeConfig.java | 29 ++ .../confluence/configuration/NameConfig.java | 30 ++ .../configuration/Oauth2Config.java | 36 ++ .../configuration/ProjectConfig.java | 20 ++ .../configuration/StatusConfig.java | 29 ++ .../exception/BadRequestException.java | 25 ++ .../exception/UnAuthorizedException.java | 25 ++ .../source/confluence/models/IssueBean.java | 125 +++++++ .../confluence/models/SearchResults.java | 39 +++ .../confluence/rest/BasicAuthInterceptor.java | 42 +++ .../confluence/rest/ConfluenceRestClient.java | 148 ++++++++ .../rest/CustomRestTemplateConfig.java | 42 +++ .../rest/OAuth2RequestInterceptor.java | 36 ++ .../rest/auth/ConfluenceAuthConfig.java | 34 ++ .../rest/auth/ConfluenceAuthFactory.java | 41 +++ .../rest/auth/ConfluenceBasicAuthConfig.java | 45 +++ .../rest/auth/ConfluenceOauthConfig.java | 183 ++++++++++ .../confluence/utils/AddressValidation.java | 60 ++++ .../utils/ConfluenceConfigHelper.java | 120 +++++++ .../utils/ConfluenceContentType.java | 26 ++ .../source/confluence/utils/Constants.java | 34 ++ .../source/confluence/utils/JqlConstants.java | 30 ++ .../confluence/ConfluenceClientTest.java | 149 ++++++++ .../ConfluenceConfigHelperTest.java | 173 ++++++++++ .../confluence/ConfluenceItemInfoTest.java | 109 ++++++ .../confluence/ConfluenceIteratorTest.java | 201 +++++++++++ .../confluence/ConfluenceServiceTest.java | 325 ++++++++++++++++++ .../ConfluenceSourceConfigTest.java | 137 ++++++++ .../confluence/ConfluenceSourceTest.java | 118 +++++++ .../exception/BadRequestExceptionTest.java | 65 ++++ .../exception/UnAuthorizedExceptionTest.java | 65 ++++ .../confluence/models/IssueBeanTest.java | 131 +++++++ .../confluence/models/SearchResultsTest.java | 98 ++++++ .../rest/BasicAuthInterceptorTest.java | 80 +++++ .../rest/ConfluenceRestClientTest.java | 150 ++++++++ .../rest/CustomRestTemplateConfigTest.java | 105 ++++++ .../rest/OAuth2RequestInterceptorTest.java | 66 ++++ .../rest/auth/ConfluenceAuthFactoryTest.java | 74 ++++ .../rest/auth/JiraBasicAuthConfigTest.java | 48 +++ .../rest/auth/JiraOauthConfigTest.java | 187 ++++++++++ .../utils/AddressValidationTest.java | 105 ++++++ .../utils/ConfluenceContentTypeTest.java | 36 ++ .../utils/MockPluginConfigVariableImpl.java | 44 +++ .../resources/basic-auth-jira-pipeline.yaml | 6 + .../resources/oauth2-auth-jira-pipeline.yaml | 6 + .../source/jira/rest/JiraRestClient.java | 5 +- settings.gradle | 1 + 58 files changed, 4613 insertions(+), 2 deletions(-) create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/README.md create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/build.gradle create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceClient.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceItemInfo.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceIterator.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSource.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceConfig.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/AuthenticationConfig.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/BasicConfig.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/FilterConfig.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/IssueTypeConfig.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/NameConfig.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/Oauth2Config.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/ProjectConfig.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/StatusConfig.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/exception/BadRequestException.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/exception/UnAuthorizedException.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/IssueBean.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/SearchResults.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/BasicAuthInterceptor.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClient.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/CustomRestTemplateConfig.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/OAuth2RequestInterceptor.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceAuthConfig.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceAuthFactory.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceBasicAuthConfig.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceOauthConfig.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/AddressValidation.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/ConfluenceConfigHelper.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/ConfluenceContentType.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/Constants.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/JqlConstants.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceClientTest.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceConfigHelperTest.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceItemInfoTest.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceIteratorTest.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceServiceTest.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceConfigTest.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceTest.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/exception/BadRequestExceptionTest.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/exception/UnAuthorizedExceptionTest.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/IssueBeanTest.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/SearchResultsTest.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/BasicAuthInterceptorTest.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClientTest.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/CustomRestTemplateConfigTest.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/OAuth2RequestInterceptorTest.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceAuthFactoryTest.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/JiraBasicAuthConfigTest.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/JiraOauthConfigTest.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/utils/AddressValidationTest.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/utils/ConfluenceContentTypeTest.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/utils/MockPluginConfigVariableImpl.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/test/resources/basic-auth-jira-pipeline.yaml create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/test/resources/oauth2-auth-jira-pipeline.yaml diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/README.md b/data-prepper-plugins/saas-source-plugins/confluence-source/README.md new file mode 100644 index 0000000000..a5f5d962c1 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/README.md @@ -0,0 +1,9 @@ +# Metrics + +### Counter + +- `issuesRequested`: measures total number of issue Requests sent. + +### Timer + +- `requestProcessDuration`: measures latency of requests processed by the jira source plugin. diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/build.gradle b/data-prepper-plugins/saas-source-plugins/confluence-source/build.gradle new file mode 100644 index 0000000000..1cf3854ab4 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/build.gradle @@ -0,0 +1,33 @@ +plugins { + id 'java' +} + +dependencies { + + implementation project(path: ':data-prepper-plugins:saas-source-plugins:source-crawler') + implementation project(path: ':data-prepper-api') + implementation project(path: ':data-prepper-plugins:aws-plugin-api') + implementation project(path: ':data-prepper-plugins:buffer-common') + implementation project(path: ':data-prepper-plugins:common') + + implementation libs.commons.io + implementation 'io.micrometer:micrometer-core' + implementation 'com.fasterxml.jackson.core:jackson-core' + implementation 'com.fasterxml.jackson.core:jackson-databind' + implementation 'javax.inject:javax.inject:1' + implementation("org.springframework:spring-web:${libs.versions.spring.get()}") + + implementation 'org.projectlombok:lombok:1.18.30' + annotationProcessor 'org.projectlombok:lombok:1.18.30' + + testImplementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.4' + testImplementation project(path: ':data-prepper-test-common') + + implementation(libs.spring.context) { + exclude group: 'commons-logging', module: 'commons-logging' + } +} + +test { + useJUnitPlatform() +} \ No newline at end of file diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceClient.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceClient.java new file mode 100644 index 0000000000..68874f8587 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceClient.java @@ -0,0 +1,146 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.annotations.VisibleForTesting; +import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSet; +import org.opensearch.dataprepper.model.buffer.Buffer; +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.event.EventType; +import org.opensearch.dataprepper.model.event.JacksonEvent; +import org.opensearch.dataprepper.model.record.Record; +import org.opensearch.dataprepper.plugins.source.source_crawler.base.CrawlerClient; +import org.opensearch.dataprepper.plugins.source.source_crawler.base.CrawlerSourceConfig; +import org.opensearch.dataprepper.plugins.source.source_crawler.base.PluginExecutorServiceProvider; +import org.opensearch.dataprepper.plugins.source.source_crawler.coordination.state.SaasWorkerProgressState; +import org.opensearch.dataprepper.plugins.source.source_crawler.model.ItemInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Named; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import static java.util.concurrent.CompletableFuture.supplyAsync; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.PROJECT; + +/** + * This class represents a Jira client. + */ +@Named +public class ConfluenceClient implements CrawlerClient { + + private static final Logger log = LoggerFactory.getLogger(ConfluenceClient.class); + private ObjectMapper objectMapper = new ObjectMapper(); + private Instant lastPollTime; + private final ConfluenceService service; + private final ConfluenceIterator confluenceIterator; + private final ExecutorService executorService; + private final CrawlerSourceConfig configuration; + private final int bufferWriteTimeoutInSeconds = 10; + + public ConfluenceClient(ConfluenceService service, + ConfluenceIterator confluenceIterator, + PluginExecutorServiceProvider executorServiceProvider, + ConfluenceSourceConfig sourceConfig) { + this.service = service; + this.confluenceIterator = confluenceIterator; + this.executorService = executorServiceProvider.get(); + this.configuration = sourceConfig; + } + + @Override + public Iterator listItems() { + confluenceIterator.initialize(lastPollTime); + return confluenceIterator; + } + + @Override + public void setLastPollTime(Instant lastPollTime) { + log.trace("Setting the lastPollTime: {}", lastPollTime); + this.lastPollTime = lastPollTime; + } + + @VisibleForTesting + public void injectObjectMapper(ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + } + + @Override + public void executePartition(SaasWorkerProgressState state, + Buffer> buffer, + AcknowledgementSet acknowledgementSet) { + log.trace("Executing the partition: {} with {} ticket(s)", + state.getKeyAttributes(), state.getItemIds().size()); + List itemIds = state.getItemIds(); + Map keyAttributes = state.getKeyAttributes(); + String project = (String) keyAttributes.get(PROJECT); + Instant eventTime = state.getExportStartTime(); + List itemInfos = new ArrayList<>(); + for (String itemId : itemIds) { + if (itemId == null) { + continue; + } + ItemInfo itemInfo = ConfluenceItemInfo.builder() + .withItemId(itemId) + .withId(itemId) + .withProject(project) + .withEventTime(eventTime) + .withMetadata(keyAttributes).build(); + itemInfos.add(itemInfo); + } + + String eventType = EventType.DOCUMENT.toString(); + List> recordsToWrite = itemInfos + .parallelStream() + .map(t -> (Supplier) (() -> service.getIssue(t.getId()))) + .map(supplier -> supplyAsync(supplier, this.executorService)) + .map(CompletableFuture::join) + .map(ticketJson -> { + try { + return objectMapper.readValue(ticketJson, new TypeReference<>() { + }); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + }) + .map(t -> (Event) JacksonEvent.builder() + .withEventType(eventType) + .withData(t) + .build()) + .map(event -> new Record<>(event)) + .collect(Collectors.toList()); + + try { + if (configuration.isAcknowledgments()) { + recordsToWrite.forEach(eventRecord -> acknowledgementSet.add(eventRecord.getData())); + buffer.writeAll(recordsToWrite, (int) Duration.ofSeconds(bufferWriteTimeoutInSeconds).toMillis()); + acknowledgementSet.complete(); + } else { + buffer.writeAll(recordsToWrite, (int) Duration.ofSeconds(bufferWriteTimeoutInSeconds).toMillis()); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + + } +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceItemInfo.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceItemInfo.java new file mode 100644 index 0000000000..15aa46009c --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceItemInfo.java @@ -0,0 +1,158 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence; + +import lombok.Getter; +import lombok.Setter; +import org.opensearch.dataprepper.plugins.source.confluence.models.IssueBean; +import org.opensearch.dataprepper.plugins.source.confluence.utils.ConfluenceContentType; +import org.opensearch.dataprepper.plugins.source.confluence.utils.Constants; +import org.opensearch.dataprepper.plugins.source.source_crawler.model.ItemInfo; + +import java.time.Instant; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.CREATED; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.ISSUE_KEY; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.PROJECT_KEY; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.PROJECT_NAME; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.UPDATED; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants._ISSUE; + +@Setter +@Getter +public class ConfluenceItemInfo implements ItemInfo { + private String project; + private String issueType; + private String id; + private String itemId; + private Map metadata; + private Instant eventTime; + + public ConfluenceItemInfo(String id, + String itemId, + String project, + String issueType, + Map metadata, + Instant eventTime + ) { + this.id = id; + this.project = project; + this.issueType = issueType; + this.itemId = itemId; + this.metadata = metadata; + this.eventTime = eventTime; + } + + public static ConfluenceItemInfoBuilder builder() { + return new ConfluenceItemInfoBuilder(); + } + + @Override + public String getPartitionKey() { + return project + "|" + issueType + "|" + UUID.randomUUID(); + } + + @Override + public String getId() { + return id; + } + + @Override + public Map getKeyAttributes() { + return Map.of(Constants.PROJECT, project); + } + + @Override + public Instant getLastModifiedAt() { + long updatedAtMillis = getMetadataField(Constants.UPDATED); + long createdAtMillis = getMetadataField(Constants.CREATED); + return createdAtMillis > updatedAtMillis ? + Instant.ofEpochMilli(createdAtMillis) : Instant.ofEpochMilli(updatedAtMillis); + } + + private Long getMetadataField(String fieldName) { + Object value = this.metadata.get(fieldName); + if (value == null) { + return 0L; + } else if (value instanceof Long) { + return (Long) value; + } else if (value instanceof String) { + try { + return Long.parseLong((String) value); + } catch (Exception e) { + return 0L; + } + } + return 0L; + } + + public static class ConfluenceItemInfoBuilder { + private Map metadata; + private Instant eventTime; + private String id; + private String itemId; + private String project; + private String issueType; + + public ConfluenceItemInfoBuilder() { + } + + public ConfluenceItemInfo build() { + return new ConfluenceItemInfo(id, itemId, project, issueType, metadata, eventTime); + } + + public ConfluenceItemInfoBuilder withMetadata(Map metadata) { + this.metadata = metadata; + return this; + } + + public ConfluenceItemInfoBuilder withEventTime(Instant eventTime) { + this.eventTime = eventTime; + return this; + } + + public ConfluenceItemInfoBuilder withItemId(String itemId) { + this.itemId = itemId; + return this; + } + + public ConfluenceItemInfoBuilder withId(String id) { + this.id = id; + return this; + } + + public ConfluenceItemInfoBuilder withProject(String project) { + this.project = project; + return this; + } + + public ConfluenceItemInfoBuilder withIssueBean(IssueBean issue) { + Map issueMetadata = new HashMap<>(); + issueMetadata.put(PROJECT_KEY, issue.getProject()); + issueMetadata.put(PROJECT_NAME, issue.getProjectName()); + issueMetadata.put(CREATED, issue.getCreatedTimeMillis()); + issueMetadata.put(UPDATED, issue.getUpdatedTimeMillis()); + issueMetadata.put(ISSUE_KEY, issue.getKey()); + issueMetadata.put(ConfluenceService.CONTENT_TYPE, ConfluenceContentType.ISSUE.getType()); + + this.project = issue.getProject(); + this.id = issue.getKey(); + this.issueType = ConfluenceContentType.ISSUE.getType(); + this.itemId = _ISSUE + issueMetadata.get(PROJECT_KEY) + "-" + issue.getKey(); + this.metadata = issueMetadata; + return this; + } + } + +} \ No newline at end of file diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceIterator.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceIterator.java new file mode 100644 index 0000000000..bc67568083 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceIterator.java @@ -0,0 +1,124 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence; + + +import com.google.common.annotations.VisibleForTesting; +import lombok.Setter; +import org.opensearch.dataprepper.plugins.source.source_crawler.base.PluginExecutorServiceProvider; +import org.opensearch.dataprepper.plugins.source.source_crawler.model.ItemInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Named; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; + +@Named +public class ConfluenceIterator implements Iterator { + + private static final int HAS_NEXT_TIMEOUT = 60; + private static final Logger log = LoggerFactory.getLogger(ConfluenceIterator.class); + @Setter + private long crawlerQWaitTimeMillis = 2000; + private Queue itemInfoQueue; + private Instant lastPollTime; + private boolean firstTime = true; + private final List> futureList = new ArrayList<>(); + private final ConfluenceSourceConfig sourceConfig; + private final ConfluenceService service; + private final ExecutorService crawlerTaskExecutor; + + public ConfluenceIterator(final ConfluenceService service, + PluginExecutorServiceProvider executorServiceProvider, + ConfluenceSourceConfig sourceConfig) { + this.service = service; + this.crawlerTaskExecutor = executorServiceProvider.get(); + this.sourceConfig = sourceConfig; + } + + @Override + public boolean hasNext() { + if (firstTime) { + log.trace("Crawling has been started"); + startCrawlerThreads(); + firstTime = false; + } + int timeout = HAS_NEXT_TIMEOUT; + while (isCrawlerRunning() && itemInfoQueue.isEmpty() && timeout > 0) { + try { + log.trace("Waiting for crawler queue to be filled for next {} seconds", timeout); + Thread.sleep(crawlerQWaitTimeMillis); + timeout--; + } catch (InterruptedException e) { + log.error("An exception has occurred while checking for the next document in crawling queue"); + Thread.currentThread().interrupt(); + } + } + return !this.itemInfoQueue.isEmpty(); + } + + private boolean isCrawlerRunning() { + boolean isRunning = false; + if (!futureList.isEmpty()) { + for (Future future : futureList) { + if (!future.isDone()) { + isRunning = true; + break; + } + } + } + return isRunning; + } + + private void startCrawlerThreads() { + futureList.add(crawlerTaskExecutor.submit(() -> + service.getJiraEntities(sourceConfig, lastPollTime, itemInfoQueue), false)); + } + + @Override + public ItemInfo next() { + if (hasNext()) { + return this.itemInfoQueue.remove(); + } else { + throw new NoSuchElementException(); + } + } + + /** + * Initialize. + * + * @param jiraChangeLogToken the jira change log token + */ + public void initialize(Instant jiraChangeLogToken) { + this.itemInfoQueue = new ConcurrentLinkedQueue<>(); + this.lastPollTime = jiraChangeLogToken; + this.firstTime = true; + } + + @VisibleForTesting + public List> showFutureList() { + return futureList; + } + + @VisibleForTesting + public Queue showItemInfoQueue() { + return itemInfoQueue; + } + +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java new file mode 100644 index 0000000000..e1dfe6e481 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java @@ -0,0 +1,217 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence; + +import io.micrometer.core.instrument.Counter; +import lombok.extern.slf4j.Slf4j; +import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.plugins.source.confluence.exception.BadRequestException; +import org.opensearch.dataprepper.plugins.source.confluence.models.IssueBean; +import org.opensearch.dataprepper.plugins.source.confluence.models.SearchResults; +import org.opensearch.dataprepper.plugins.source.confluence.rest.ConfluenceRestClient; +import org.opensearch.dataprepper.plugins.source.confluence.utils.ConfluenceConfigHelper; +import org.opensearch.dataprepper.plugins.source.source_crawler.model.ItemInfo; +import org.springframework.util.CollectionUtils; + +import javax.inject.Named; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Queue; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.UPDATED; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.CLOSING_ROUND_BRACKET; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.DELIMITER; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.GREATER_THAN_EQUALS; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.ISSUE_TYPE_IN; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.ISSUE_TYPE_NOT_IN; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.PREFIX; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.PROJECT_IN; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.PROJECT_NOT_IN; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.STATUS_IN; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.STATUS_NOT_IN; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.SUFFIX; + + +/** + * Service class for interactive external Atlassian jira SaaS service and fetch required details using their rest apis. + */ + +@Slf4j +@Named +public class ConfluenceService { + + + public static final String CONTENT_TYPE = "ContentType"; + private static final String SEARCH_RESULTS_FOUND = "searchResultsFound"; + + private final ConfluenceSourceConfig confluenceSourceConfig; + private final ConfluenceRestClient confluenceRestClient; + private final Counter searchResultsFoundCounter; + private final PluginMetrics jiraPluginMetrics = PluginMetrics.fromNames("jiraService", "aws"); + + + public ConfluenceService(ConfluenceSourceConfig confluenceSourceConfig, ConfluenceRestClient confluenceRestClient) { + this.confluenceSourceConfig = confluenceSourceConfig; + this.confluenceRestClient = confluenceRestClient; + this.searchResultsFoundCounter = jiraPluginMetrics.counter(SEARCH_RESULTS_FOUND); + } + + /** + * Get jira entities. + * + * @param configuration the configuration. + * @param timestamp timestamp. + */ + public void getJiraEntities(ConfluenceSourceConfig configuration, Instant timestamp, Queue itemInfoQueue) { + log.trace("Started to fetch entities"); + searchForNewTicketsAndAddToQueue(configuration, timestamp, itemInfoQueue); + log.trace("Creating item information and adding in queue"); + } + + public String getIssue(String issueKey) { + return confluenceRestClient.getIssue(issueKey); + } + + /** + * Method for building Issue Item Info. + * + * @param configuration Input Parameter + * @param timestamp Input Parameter + */ + private void searchForNewTicketsAndAddToQueue(ConfluenceSourceConfig configuration, Instant timestamp, + Queue itemInfoQueue) { + log.trace("Looking for Add/Modified tickets with a Search API call"); + StringBuilder jql = createIssueFilterCriteria(configuration, timestamp); + int total; + int startAt = 0; + do { + SearchResults searchIssues = confluenceRestClient.getAllIssues(jql, startAt, configuration); + List issueList = new ArrayList<>(searchIssues.getIssues()); + total = searchIssues.getTotal(); + startAt += searchIssues.getIssues().size(); + addItemsToQueue(issueList, itemInfoQueue); + } while (startAt < total); + searchResultsFoundCounter.increment(total); + log.info("Number of tickets found in search api call: {}", total); + } + + /** + * Add items to queue. + * + * @param issueList Issue list. + * @param itemInfoQueue Item info queue. + */ + private void addItemsToQueue(List issueList, Queue itemInfoQueue) { + issueList.forEach(issue -> { + itemInfoQueue.add(ConfluenceItemInfo.builder().withEventTime(Instant.now()).withIssueBean(issue).build()); + }); + } + + + /** + * Method for creating Issue Filter Criteria. + * + * @param configuration Input Parameter + * @param ts Input Parameter + * @return String Builder + */ + private StringBuilder createIssueFilterCriteria(ConfluenceSourceConfig configuration, Instant ts) { + + log.info("Creating issue filter criteria"); + if (!CollectionUtils.isEmpty(ConfluenceConfigHelper.getProjectNameIncludeFilter(configuration)) || !CollectionUtils.isEmpty(ConfluenceConfigHelper.getProjectNameExcludeFilter(configuration))) { + validateProjectFilters(configuration); + } + StringBuilder jiraQl = new StringBuilder(UPDATED + GREATER_THAN_EQUALS + ts.toEpochMilli()); + if (!CollectionUtils.isEmpty(ConfluenceConfigHelper.getProjectNameIncludeFilter(configuration))) { + jiraQl.append(PROJECT_IN).append(ConfluenceConfigHelper.getProjectNameIncludeFilter(configuration).stream() + .collect(Collectors.joining(DELIMITER, PREFIX, SUFFIX))) + .append(CLOSING_ROUND_BRACKET); + } + if (!CollectionUtils.isEmpty(ConfluenceConfigHelper.getProjectNameExcludeFilter(configuration))) { + jiraQl.append(PROJECT_NOT_IN).append(ConfluenceConfigHelper.getProjectNameExcludeFilter(configuration).stream() + .collect(Collectors.joining(DELIMITER, PREFIX, SUFFIX))) + .append(CLOSING_ROUND_BRACKET); + } + if (!CollectionUtils.isEmpty(ConfluenceConfigHelper.getIssueTypeIncludeFilter(configuration))) { + jiraQl.append(ISSUE_TYPE_IN).append(ConfluenceConfigHelper.getIssueTypeIncludeFilter(configuration).stream() + .collect(Collectors.joining(DELIMITER, PREFIX, SUFFIX))) + .append(CLOSING_ROUND_BRACKET); + } + if (!CollectionUtils.isEmpty(ConfluenceConfigHelper.getIssueTypeExcludeFilter(configuration))) { + jiraQl.append(ISSUE_TYPE_NOT_IN).append(ConfluenceConfigHelper.getIssueTypeExcludeFilter(configuration).stream() + .collect(Collectors.joining(DELIMITER, PREFIX, SUFFIX))) + .append(CLOSING_ROUND_BRACKET); + } + if (!CollectionUtils.isEmpty(ConfluenceConfigHelper.getIssueStatusIncludeFilter(configuration))) { + jiraQl.append(STATUS_IN).append(ConfluenceConfigHelper.getIssueStatusIncludeFilter(configuration).stream() + .collect(Collectors.joining(DELIMITER, PREFIX, SUFFIX))) + .append(CLOSING_ROUND_BRACKET); + } + if (!CollectionUtils.isEmpty(ConfluenceConfigHelper.getIssueStatusExcludeFilter(configuration))) { + jiraQl.append(STATUS_NOT_IN).append(ConfluenceConfigHelper.getIssueStatusExcludeFilter(configuration).stream() + .collect(Collectors.joining(DELIMITER, PREFIX, SUFFIX))) + .append(CLOSING_ROUND_BRACKET); + } + log.error("Created issue filter criteria JiraQl query: {}", jiraQl); + return jiraQl; + } + + /** + * Method for Validating Project Filters. + * + * @param configuration Input Parameter + */ + private void validateProjectFilters(ConfluenceSourceConfig configuration) { + log.trace("Validating project filters"); + List badFilters = new ArrayList<>(); + Set includedProjects = new HashSet<>(); + List includedAndExcludedProjects = new ArrayList<>(); + Pattern regex = Pattern.compile("[^A-Z0-9]"); + ConfluenceConfigHelper.getProjectNameIncludeFilter(configuration).forEach(projectFilter -> { + Matcher matcher = regex.matcher(projectFilter); + includedProjects.add(projectFilter); + if (matcher.find() || projectFilter.length() <= 1 || projectFilter.length() > 10) { + badFilters.add(projectFilter); + } + }); + ConfluenceConfigHelper.getProjectNameExcludeFilter(configuration).forEach(projectFilter -> { + Matcher matcher = regex.matcher(projectFilter); + if (includedProjects.contains(projectFilter)) { + includedAndExcludedProjects.add(projectFilter); + } + if (matcher.find() || projectFilter.length() <= 1 || projectFilter.length() > 10) { + badFilters.add(projectFilter); + } + }); + if (!badFilters.isEmpty()) { + String filters = String.join("\"" + badFilters + "\"", ", "); + log.error("One or more invalid project keys found in filter configuration: {}", badFilters); + throw new BadRequestException("Bad request exception occurred " + + "Invalid project key found in filter configuration for " + + filters); + } + if (!includedAndExcludedProjects.isEmpty()) { + String filters = String.join("\"" + includedAndExcludedProjects + "\"", ", "); + log.error("One or more project keys found in both include and exclude: {}", includedAndExcludedProjects); + throw new BadRequestException("Bad request exception occurred " + + "Project filters is invalid because the following projects are listed in both include and exclude" + + filters); + } + + } + +} \ No newline at end of file diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSource.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSource.java new file mode 100644 index 0000000000..69294be228 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSource.java @@ -0,0 +1,78 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence; + + +import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSetManager; +import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; +import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; +import org.opensearch.dataprepper.model.buffer.Buffer; +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.plugin.PluginFactory; +import org.opensearch.dataprepper.model.record.Record; +import org.opensearch.dataprepper.model.source.Source; +import org.opensearch.dataprepper.plugins.source.confluence.rest.auth.ConfluenceAuthConfig; +import org.opensearch.dataprepper.plugins.source.confluence.utils.ConfluenceConfigHelper; +import org.opensearch.dataprepper.plugins.source.source_crawler.CrawlerApplicationContextMarker; +import org.opensearch.dataprepper.plugins.source.source_crawler.base.Crawler; +import org.opensearch.dataprepper.plugins.source.source_crawler.base.CrawlerSourcePlugin; +import org.opensearch.dataprepper.plugins.source.source_crawler.base.PluginExecutorServiceProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.PLUGIN_NAME; + + +/** + * JiraConnector connector entry point. + */ + +@DataPrepperPlugin(name = PLUGIN_NAME, + pluginType = Source.class, + pluginConfigurationType = ConfluenceSourceConfig.class, + packagesToScan = {CrawlerApplicationContextMarker.class, ConfluenceSource.class} +) +public class ConfluenceSource extends CrawlerSourcePlugin { + + private static final Logger log = LoggerFactory.getLogger(ConfluenceSource.class); + private final ConfluenceSourceConfig confluenceSourceConfig; + private final ConfluenceAuthConfig jiraOauthConfig; + + @DataPrepperPluginConstructor + public ConfluenceSource(final PluginMetrics pluginMetrics, + final ConfluenceSourceConfig confluenceSourceConfig, + final ConfluenceAuthConfig jiraOauthConfig, + final PluginFactory pluginFactory, + final AcknowledgementSetManager acknowledgementSetManager, + Crawler crawler, + PluginExecutorServiceProvider executorServiceProvider) { + super(PLUGIN_NAME, pluginMetrics, confluenceSourceConfig, pluginFactory, acknowledgementSetManager, crawler, executorServiceProvider); + log.info("Creating Jira Source Plugin"); + this.confluenceSourceConfig = confluenceSourceConfig; + this.jiraOauthConfig = jiraOauthConfig; + } + + @Override + public void start(Buffer> buffer) { + log.info("Starting Jira Source Plugin... "); + ConfluenceConfigHelper.validateConfig(confluenceSourceConfig); + jiraOauthConfig.initCredentials(); + super.start(buffer); + } + + @Override + public void stop() { + log.info("Stopping Jira Source Plugin"); + super.stop(); + } + +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceConfig.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceConfig.java new file mode 100644 index 0000000000..57afe256d2 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceConfig.java @@ -0,0 +1,67 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.Valid; +import lombok.Getter; +import org.opensearch.dataprepper.plugins.source.confluence.configuration.AuthenticationConfig; +import org.opensearch.dataprepper.plugins.source.confluence.configuration.FilterConfig; +import org.opensearch.dataprepper.plugins.source.source_crawler.base.CrawlerSourceConfig; + +import java.util.List; + +@Getter +public class ConfluenceSourceConfig implements CrawlerSourceConfig { + + private static final int DEFAULT_BATCH_SIZE = 50; + + /** + * Jira account url + */ + @JsonProperty("hosts") + private List hosts; + + /** + * Authentication Config to Access Jira + */ + @JsonProperty("authentication") + @Valid + private AuthenticationConfig authenticationConfig; + + /** + * Batch size for fetching tickets + */ + @JsonProperty("batch_size") + private int batchSize = DEFAULT_BATCH_SIZE; + + + /** + * Filter Config to filter what tickets get ingested + */ + @JsonProperty("filter") + private FilterConfig filterConfig; + + + /** + * Boolean property indicating end to end acknowledgments state + */ + @JsonProperty("acknowledgments") + private boolean acknowledgments = false; + + public String getAccountUrl() { + return this.getHosts().get(0); + } + + public String getAuthType() { + return this.getAuthenticationConfig().getAuthType(); + } +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/AuthenticationConfig.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/AuthenticationConfig.java new file mode 100644 index 0000000000..d5632f5f2e --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/AuthenticationConfig.java @@ -0,0 +1,45 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.configuration; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.Valid; +import jakarta.validation.constraints.AssertTrue; +import lombok.Getter; + +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.BASIC; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.OAUTH2; + +@Getter +public class AuthenticationConfig { + @JsonProperty("basic") + @Valid + private BasicConfig basicConfig; + + @JsonProperty("oauth2") + @Valid + private Oauth2Config oauth2Config; + + @AssertTrue(message = "Authentication config should have either basic or oauth2") + private boolean isValidAuthenticationConfig() { + boolean hasBasic = basicConfig != null; + boolean hasOauth = oauth2Config != null; + return hasBasic ^ hasOauth; + } + + public String getAuthType() { + if (basicConfig != null) { + return BASIC; + } else { + return OAUTH2; + } + } +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/BasicConfig.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/BasicConfig.java new file mode 100644 index 0000000000..1cbb6de933 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/BasicConfig.java @@ -0,0 +1,29 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.configuration; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.constraints.AssertTrue; +import lombok.Getter; + +@Getter +public class BasicConfig { + @JsonProperty("username") + private String username; + + @JsonProperty("password") + private String password; + + @AssertTrue(message = "Username and Password are both required for Basic Auth") + private boolean isBasicConfigValid() { + return username != null && password != null; + } +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/FilterConfig.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/FilterConfig.java new file mode 100644 index 0000000000..040ce85107 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/FilterConfig.java @@ -0,0 +1,26 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.configuration; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Getter; + +@Getter +public class FilterConfig { + @JsonProperty("project") + private ProjectConfig projectConfig; + + @JsonProperty("status") + private StatusConfig statusConfig; + + @JsonProperty("issue_type") + private IssueTypeConfig issueTypeConfig; +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/IssueTypeConfig.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/IssueTypeConfig.java new file mode 100644 index 0000000000..56b9ff9208 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/IssueTypeConfig.java @@ -0,0 +1,29 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.configuration; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.constraints.Size; +import lombok.Getter; + +import java.util.ArrayList; +import java.util.List; + +@Getter +public class IssueTypeConfig { + @JsonProperty("include") + @Size(max = 1000, message = "Issue type filter should not be more than 1000") + private List include = new ArrayList<>(); + + @JsonProperty("exclude") + @Size(max = 1000, message = "Issue type filter should not be more than 1000") + private List exclude = new ArrayList<>(); +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/NameConfig.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/NameConfig.java new file mode 100644 index 0000000000..8982d69cf2 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/NameConfig.java @@ -0,0 +1,30 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + + +package org.opensearch.dataprepper.plugins.source.confluence.configuration; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.constraints.Size; +import lombok.Getter; + +import java.util.ArrayList; +import java.util.List; + +@Getter +public class NameConfig { + @JsonProperty("include") + @Size(max = 1000, message = "Project name type filter should not be more than 1000") + private List include = new ArrayList<>(); + + @JsonProperty("exclude") + @Size(max = 1000, message = "Project name type filter should not be more than 1000") + private List exclude = new ArrayList<>(); +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/Oauth2Config.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/Oauth2Config.java new file mode 100644 index 0000000000..22daa7841b --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/Oauth2Config.java @@ -0,0 +1,36 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.configuration; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.constraints.AssertTrue; +import lombok.Getter; +import org.opensearch.dataprepper.model.plugin.PluginConfigVariable; + +@Getter +public class Oauth2Config { + @JsonProperty("client_id") + private String clientId; + + @JsonProperty("client_secret") + private String clientSecret; + + @JsonProperty("access_token") + private PluginConfigVariable accessToken; + + @JsonProperty("refresh_token") + private PluginConfigVariable refreshToken; + + @AssertTrue(message = "Client ID, Client Secret, Access Token, and Refresh Token are both required for Oauth2") + private boolean isOauth2ConfigValid() { + return clientId != null && clientSecret != null && accessToken != null && refreshToken != null; + } +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/ProjectConfig.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/ProjectConfig.java new file mode 100644 index 0000000000..631c7ae7bf --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/ProjectConfig.java @@ -0,0 +1,20 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.configuration; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Getter; + +@Getter +public class ProjectConfig { + @JsonProperty("key") + private NameConfig nameConfig; +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/StatusConfig.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/StatusConfig.java new file mode 100644 index 0000000000..3dde46bfd9 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/StatusConfig.java @@ -0,0 +1,29 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.configuration; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.constraints.Size; +import lombok.Getter; + +import java.util.ArrayList; +import java.util.List; + +@Getter +public class StatusConfig { + @JsonProperty("include") + @Size(max = 1000, message = "status type filter should not be more than 1000") + private List include = new ArrayList<>(); + + @JsonProperty("exclude") + @Size(max = 1000, message = "status type filter should not be more than 1000") + private List exclude = new ArrayList<>(); +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/exception/BadRequestException.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/exception/BadRequestException.java new file mode 100644 index 0000000000..30a4dd36e2 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/exception/BadRequestException.java @@ -0,0 +1,25 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.exception; + +/** + * Exception to indicate a bad REST call has been made. + * It could either be caused by bad user inputs or wrong url construction in the logic. + */ +public final class BadRequestException extends RuntimeException { + public BadRequestException(final String message, final Throwable throwable) { + super(message, throwable); + } + + public BadRequestException(final String message) { + super(message); + } +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/exception/UnAuthorizedException.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/exception/UnAuthorizedException.java new file mode 100644 index 0000000000..48095d3e45 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/exception/UnAuthorizedException.java @@ -0,0 +1,25 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.exception; + +/** + * Exception to indicate unauthorized access. + * It could either be caused by invalid credentials supplied by the user or failed renew the credentials. + */ +public final class UnAuthorizedException extends RuntimeException { + public UnAuthorizedException(final String message, final Throwable throwable) { + super(message, throwable); + } + + public UnAuthorizedException(final String message) { + super(message); + } +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/IssueBean.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/IssueBean.java new file mode 100644 index 0000000000..57b6f49774 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/IssueBean.java @@ -0,0 +1,125 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.models; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Getter; +import lombok.Setter; + +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Date; +import java.util.Map; +import java.util.Objects; +import java.util.regex.Pattern; + +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.CREATED; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.KEY; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.NAME; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.PROJECT; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.UPDATED; + + +public class IssueBean { + + /** + * Expand options that include additional issue details in the response. + */ + @Getter + @Setter + @JsonProperty("expand") + private String expand = null; + /** + * The ID of the issue. + */ + @Getter + @Setter + @JsonProperty("id") + private String id = null; + /** + * The URL of the issue details. + */ + @Getter + @Setter + @JsonProperty("self") + private String self = null; + /** + * The key of the issue. + */ + @Getter + @Setter + @JsonProperty("key") + private String key = null; + @Getter + @Setter + @JsonProperty("renderedFields") + private Map renderedFields = null; + @Getter + @Setter + @JsonProperty("properties") + private Map properties = null; + @Getter + @Setter + @JsonProperty("names") + private Map names = null; + @Getter + @Setter + @JsonProperty("fields") + private Map fields = null; + @JsonIgnore + private final Pattern JiraDateTimePattern = Pattern.compile( + "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}[-+]\\d{4}$"); + @JsonIgnore + private final DateTimeFormatter offsetDateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ"); + + @JsonIgnore + public String getProject() { + if (fields != null && Objects.nonNull(((Map) fields.get(PROJECT)).get(KEY))) { + return ((Map) fields.get(PROJECT)).get(KEY).toString(); + } + return null; + } + + @JsonIgnore + public String getProjectName() { + if (fields != null && Objects.nonNull(((Map) fields.get(PROJECT)).get(NAME))) { + return ((Map) fields.get(PROJECT)).get(NAME).toString(); + } + return null; + } + + @JsonIgnore + public long getCreatedTimeMillis() { + return getGivenDateField(CREATED); + } + + @JsonIgnore + public long getUpdatedTimeMillis() { + return getGivenDateField(UPDATED); + } + + @JsonIgnore + private long getGivenDateField(String dateTimeFieldToPull) { + long dateTimeField = 0; + + if (fields != null && Objects.nonNull(fields.get(dateTimeFieldToPull)) && JiraDateTimePattern.matcher(fields.get(dateTimeFieldToPull) + .toString()).matches()) { + String charSequence = fields.get(dateTimeFieldToPull).toString(); + OffsetDateTime offsetDateTime = OffsetDateTime.parse(charSequence, offsetDateTimeFormatter); + new Date(offsetDateTime.toInstant().toEpochMilli()); + dateTimeField = offsetDateTime.toEpochSecond() * 1000; + } + return dateTimeField; + } + + +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/SearchResults.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/SearchResults.java new file mode 100644 index 0000000000..17fd06803c --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/SearchResults.java @@ -0,0 +1,39 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + + +package org.opensearch.dataprepper.plugins.source.confluence.models; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Getter; + +import java.util.List; + +/** + * The result of a JQL search. + */ +@Getter +public class SearchResults { + @JsonProperty("expand") + private String expand = null; + + @JsonProperty("startAt") + private Integer startAt = null; + + @JsonProperty("maxResults") + private Integer maxResults = null; + + @JsonProperty("total") + private Integer total = null; + + @JsonProperty("issues") + private List issues = null; + +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/BasicAuthInterceptor.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/BasicAuthInterceptor.java new file mode 100644 index 0000000000..60bada4963 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/BasicAuthInterceptor.java @@ -0,0 +1,42 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.rest; + +import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceSourceConfig; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpRequest; +import org.springframework.http.client.ClientHttpRequestExecution; +import org.springframework.http.client.ClientHttpRequestInterceptor; +import org.springframework.http.client.ClientHttpResponse; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Base64; + + +public class BasicAuthInterceptor implements ClientHttpRequestInterceptor { + private final String username; + private final String password; + + public BasicAuthInterceptor(ConfluenceSourceConfig config) { + this.username = config.getAuthenticationConfig().getBasicConfig().getUsername(); + this.password = config.getAuthenticationConfig().getBasicConfig().getPassword(); + } + + @Override + public ClientHttpResponse intercept(HttpRequest request, byte[] body, ClientHttpRequestExecution execution) throws IOException { + String auth = username + ":" + password; + byte[] encodedAuth = Base64.getEncoder().encode(auth.getBytes(StandardCharsets.US_ASCII)); + String authHeader = "Basic " + new String(encodedAuth); + request.getHeaders().set(HttpHeaders.AUTHORIZATION, authHeader); + return execution.execute(request, body); + } +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClient.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClient.java new file mode 100644 index 0000000000..9152b88a06 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClient.java @@ -0,0 +1,148 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.rest; + +import com.google.common.annotations.VisibleForTesting; +import io.micrometer.core.annotation.Timed; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.Timer; +import lombok.extern.slf4j.Slf4j; +import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceSourceConfig; +import org.opensearch.dataprepper.plugins.source.confluence.exception.BadRequestException; +import org.opensearch.dataprepper.plugins.source.confluence.exception.UnAuthorizedException; +import org.opensearch.dataprepper.plugins.source.confluence.models.SearchResults; +import org.opensearch.dataprepper.plugins.source.confluence.rest.auth.ConfluenceAuthConfig; +import org.opensearch.dataprepper.plugins.source.confluence.utils.AddressValidation; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.util.UriComponentsBuilder; + +import javax.inject.Named; +import java.net.URI; +import java.util.List; + +import static org.opensearch.dataprepper.logging.DataPrepperMarkers.NOISY; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.RETRY_ATTEMPT; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.EXPAND_FIELD; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.EXPAND_VALUE; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.JQL_FIELD; + +@Slf4j +@Named +public class ConfluenceRestClient { + + public static final String REST_API_SEARCH = "rest/api/3/search"; + public static final String REST_API_FETCH_ISSUE = "rest/api/3/issue"; + public static final String REST_API_PROJECTS = "/rest/api/3/project/search"; + public static final String FIFTY = "50"; + public static final String START_AT = "startAt"; + public static final String MAX_RESULT = "maxResults"; + public static final List RETRY_ATTEMPT_SLEEP_TIME = List.of(1, 2, 5, 10, 20, 40); + private static final String TICKET_FETCH_LATENCY_TIMER = "ticketFetchLatency"; + private static final String SEARCH_CALL_LATENCY_TIMER = "searchCallLatency"; + private static final String PROJECTS_FETCH_LATENCY_TIMER = "projectFetchLatency"; + private static final String ISSUES_REQUESTED = "issuesRequested"; + private int sleepTimeMultiplier = 1000; + private final RestTemplate restTemplate; + private final ConfluenceAuthConfig authConfig; + private final Timer ticketFetchLatencyTimer; + private final Timer searchCallLatencyTimer; + private final Timer projectFetchLatencyTimer; + private final Counter issuesRequestedCounter; + private final PluginMetrics jiraPluginMetrics = PluginMetrics.fromNames("jiraRestClient", "aws"); + + public ConfluenceRestClient(RestTemplate restTemplate, ConfluenceAuthConfig authConfig) { + this.restTemplate = restTemplate; + this.authConfig = authConfig; + + ticketFetchLatencyTimer = jiraPluginMetrics.timer(TICKET_FETCH_LATENCY_TIMER); + searchCallLatencyTimer = jiraPluginMetrics.timer(SEARCH_CALL_LATENCY_TIMER); + projectFetchLatencyTimer = jiraPluginMetrics.timer(PROJECTS_FETCH_LATENCY_TIMER); + + issuesRequestedCounter = jiraPluginMetrics.counter(ISSUES_REQUESTED); + } + + /** + * Method to get Issues. + * + * @param jql input parameter. + * @param startAt the start at + * @param configuration input parameter. + * @return InputStream input stream + */ + @Timed(SEARCH_CALL_LATENCY_TIMER) + public SearchResults getAllIssues(StringBuilder jql, int startAt, + ConfluenceSourceConfig configuration) { + + String url = authConfig.getUrl() + REST_API_SEARCH; + + URI uri = UriComponentsBuilder.fromHttpUrl(url) + .queryParam(MAX_RESULT, FIFTY) + .queryParam(START_AT, startAt) + .queryParam(JQL_FIELD, jql) + .queryParam(EXPAND_FIELD, EXPAND_VALUE) + .buildAndExpand().toUri(); + return invokeRestApi(uri, SearchResults.class).getBody(); + } + + /** + * Gets issue. + * + * @param issueKey the item info + * @return the issue + */ + @Timed(TICKET_FETCH_LATENCY_TIMER) + public String getIssue(String issueKey) { + issuesRequestedCounter.increment(); + String url = authConfig.getUrl() + REST_API_FETCH_ISSUE + "/" + issueKey; + URI uri = UriComponentsBuilder.fromHttpUrl(url).buildAndExpand().toUri(); + return invokeRestApi(uri, String.class).getBody(); + } + + private ResponseEntity invokeRestApi(URI uri, Class responseType) throws BadRequestException { + AddressValidation.validateInetAddress(AddressValidation.getInetAddress(uri.toString())); + int retryCount = 0; + while (retryCount < RETRY_ATTEMPT) { + try { + return restTemplate.getForEntity(uri, responseType); + } catch (HttpClientErrorException ex) { + HttpStatus statusCode = ex.getStatusCode(); + String statusMessage = ex.getMessage(); + log.error("An exception has occurred while getting response from Jira search API {}", ex.getMessage()); + if (statusCode == HttpStatus.FORBIDDEN) { + throw new UnAuthorizedException(statusMessage); + } else if (statusCode == HttpStatus.UNAUTHORIZED) { + log.error(NOISY, "Token expired. We will try to renew the tokens now", ex); + authConfig.renewCredentials(); + } else if (statusCode == HttpStatus.TOO_MANY_REQUESTS) { + log.error(NOISY, "Hitting API rate limit. Backing off with sleep timer.", ex); + } + try { + Thread.sleep((long) RETRY_ATTEMPT_SLEEP_TIME.get(retryCount) * sleepTimeMultiplier); + } catch (InterruptedException e) { + throw new RuntimeException("Sleep in the retry attempt got interrupted", e); + } + } + retryCount++; + } + String errorMessage = String.format("Exceeded max retry attempts. Failed to execute the Rest API call %s", uri); + log.error(errorMessage); + throw new RuntimeException(errorMessage); + } + + @VisibleForTesting + public void setSleepTimeMultiplier(int multiplier) { + sleepTimeMultiplier = multiplier; + } +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/CustomRestTemplateConfig.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/CustomRestTemplateConfig.java new file mode 100644 index 0000000000..197ede2ac2 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/CustomRestTemplateConfig.java @@ -0,0 +1,42 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.rest; + + +import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceSourceConfig; +import org.opensearch.dataprepper.plugins.source.confluence.rest.auth.ConfluenceAuthConfig; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.http.client.ClientHttpRequestInterceptor; +import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; +import org.springframework.web.client.RestTemplate; + +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.OAUTH2; + +@Configuration +public class CustomRestTemplateConfig { + + @Bean + public RestTemplate basicAuthRestTemplate(ConfluenceSourceConfig config, ConfluenceAuthConfig authConfig) { + RestTemplate restTemplate = new RestTemplate(); + restTemplate.setRequestFactory(new HttpComponentsClientHttpRequestFactory()); + ClientHttpRequestInterceptor httpInterceptor; + if (OAUTH2.equals(config.getAuthType())) { + httpInterceptor = new OAuth2RequestInterceptor(authConfig); + } else { + httpInterceptor = new BasicAuthInterceptor(config); + } + restTemplate.getInterceptors().add(httpInterceptor); + return restTemplate; + } + + +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/OAuth2RequestInterceptor.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/OAuth2RequestInterceptor.java new file mode 100644 index 0000000000..5488fd0850 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/OAuth2RequestInterceptor.java @@ -0,0 +1,36 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.rest; + +import org.opensearch.dataprepper.plugins.source.confluence.rest.auth.ConfluenceAuthConfig; +import org.opensearch.dataprepper.plugins.source.confluence.rest.auth.ConfluenceOauthConfig; +import org.springframework.http.HttpRequest; +import org.springframework.http.client.ClientHttpRequestExecution; +import org.springframework.http.client.ClientHttpRequestInterceptor; +import org.springframework.http.client.ClientHttpResponse; + +import java.io.IOException; + +public class OAuth2RequestInterceptor implements ClientHttpRequestInterceptor { + + private final ConfluenceAuthConfig config; + + public OAuth2RequestInterceptor(ConfluenceAuthConfig config) { + this.config = config; + } + + @Override + public ClientHttpResponse intercept(HttpRequest request, byte[] body, ClientHttpRequestExecution execution) throws IOException { + request.getHeaders().setBearerAuth(((ConfluenceOauthConfig) config).getAccessToken()); + return execution.execute(request, body); + } + +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceAuthConfig.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceAuthConfig.java new file mode 100644 index 0000000000..4104601261 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceAuthConfig.java @@ -0,0 +1,34 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.rest.auth; + +/** + * The interface that defines the behaviour for Jira auth configs. + */ +public interface ConfluenceAuthConfig { + + /** + * Returns the URL for the Jira instance. + * + * @return the URL for the Jira instance. + */ + String getUrl(); + + /** + * Initializes the credentials for the Jira instance. + */ + void initCredentials(); + + /** + * Renews the credentials for the Jira instance. + */ + void renewCredentials(); +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceAuthFactory.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceAuthFactory.java new file mode 100644 index 0000000000..08b4e929cb --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceAuthFactory.java @@ -0,0 +1,41 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.rest.auth; + +import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceSourceConfig; +import org.springframework.beans.factory.FactoryBean; +import org.springframework.context.annotation.Configuration; + +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.OAUTH2; + +@Configuration +public class ConfluenceAuthFactory implements FactoryBean { + + private final ConfluenceSourceConfig sourceConfig; + + public ConfluenceAuthFactory(ConfluenceSourceConfig sourceConfig) { + this.sourceConfig = sourceConfig; + } + + @Override + public ConfluenceAuthConfig getObject() { + String authType = sourceConfig.getAuthType(); + if (OAUTH2.equals(authType)) { + return new ConfluenceOauthConfig(sourceConfig); + } + return new ConfluenceBasicAuthConfig(sourceConfig); + } + + @Override + public Class getObjectType() { + return ConfluenceAuthConfig.class; + } +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceBasicAuthConfig.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceBasicAuthConfig.java new file mode 100644 index 0000000000..e400dbf7b0 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceBasicAuthConfig.java @@ -0,0 +1,45 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.rest.auth; + + +import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceSourceConfig; + +public class ConfluenceBasicAuthConfig implements ConfluenceAuthConfig { + + private String accountUrl; + private final ConfluenceSourceConfig confluenceSourceConfig; + + public ConfluenceBasicAuthConfig(ConfluenceSourceConfig confluenceSourceConfig) { + this.confluenceSourceConfig = confluenceSourceConfig; + accountUrl = confluenceSourceConfig.getAccountUrl(); + if (!accountUrl.endsWith("/")) { + accountUrl += "/"; + } + } + + @Override + public String getUrl() { + return accountUrl; + } + + @Override + public void initCredentials() { + //do nothing for basic authentication + } + + @Override + public void renewCredentials() { + //do nothing for basic authentication + } + + +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceOauthConfig.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceOauthConfig.java new file mode 100644 index 0000000000..51638b4dcb --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceOauthConfig.java @@ -0,0 +1,183 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.rest.auth; + +import lombok.Getter; +import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceSourceConfig; +import org.opensearch.dataprepper.plugins.source.confluence.configuration.Oauth2Config; +import org.opensearch.dataprepper.plugins.source.confluence.exception.UnAuthorizedException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.util.StringUtils; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.RestTemplate; + +import java.time.Instant; +import java.util.List; +import java.util.Map; + +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.RETRY_ATTEMPT; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.SLASH; + +/** + * The type Jira service. + */ + +public class ConfluenceOauthConfig implements ConfluenceAuthConfig { + + public static final String OAuth2_URL = "https://api.atlassian.com/ex/jira/"; + public static final String ACCESSIBLE_RESOURCES = "https://api.atlassian.com/oauth/token/accessible-resources"; + public static final String TOKEN_LOCATION = "https://auth.atlassian.com/oauth/token"; + + public static final String EXPIRES_IN = "expires_in"; + public static final String REFRESH_TOKEN = "refresh_token"; + public static final String ACCESS_TOKEN = "access_token"; + private static final Logger log = LoggerFactory.getLogger(ConfluenceOauthConfig.class); + RestTemplate restTemplate = new RestTemplate(); + private String url; + @Getter + private int expiresInSeconds = 0; + @Getter + private Instant expireTime = Instant.ofEpochMilli(0); + @Getter + private String accessToken; + @Getter + private String refreshToken; + private String cloudId = null; + private final String clientId; + private final String clientSecret; + private final ConfluenceSourceConfig confluenceSourceConfig; + private final Object cloudIdFetchLock = new Object(); + private final Object tokenRenewLock = new Object(); + + public ConfluenceOauthConfig(ConfluenceSourceConfig confluenceSourceConfig) { + this.confluenceSourceConfig = confluenceSourceConfig; + this.accessToken = (String) confluenceSourceConfig.getAuthenticationConfig().getOauth2Config() + .getAccessToken().getValue(); + this.refreshToken = (String) confluenceSourceConfig.getAuthenticationConfig() + .getOauth2Config().getRefreshToken().getValue(); + this.clientId = confluenceSourceConfig.getAuthenticationConfig().getOauth2Config().getClientId(); + this.clientSecret = confluenceSourceConfig.getAuthenticationConfig().getOauth2Config().getClientSecret(); + } + + public String getJiraAccountCloudId() { + log.info("Getting Jira Account Cloud ID"); + synchronized (cloudIdFetchLock) { + if (this.cloudId != null) { + //Someone else must have initialized it + return this.cloudId; + } + + int retryCount = 0; + while (retryCount < RETRY_ATTEMPT) { + retryCount++; + try { + HttpHeaders headers = new HttpHeaders(); + headers.setBearerAuth(accessToken); + headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED); + HttpEntity entity = new HttpEntity<>(headers); + ResponseEntity exchangeResponse = + restTemplate.exchange(ACCESSIBLE_RESOURCES, HttpMethod.GET, entity, Object.class); + List> listResponse = (List>) exchangeResponse.getBody(); + Map response = listResponse.get(0); + this.cloudId = (String) response.get("id"); + return this.cloudId; + } catch (HttpClientErrorException e) { + if (e.getRawStatusCode() == HttpStatus.UNAUTHORIZED.value()) { + renewCredentials(); + } + log.error("Error occurred while accessing resources: ", e); + } + } + throw new UnAuthorizedException(String.format("Access token expired. Unable to renew even after %s attempts", RETRY_ATTEMPT)); + } + } + + public void renewCredentials() { + Instant currentTime = Instant.now(); + if (expireTime.isAfter(currentTime)) { + //There is still time to renew or someone else must have already renewed it + return; + } + + synchronized (tokenRenewLock) { + if (expireTime.isAfter(currentTime)) { + //Someone else must have already renewed it + return; + } + + log.info("Renewing access token and refresh token pair for Jira Connector."); + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + String payloadTemplate = "{\"grant_type\": \"%s\", \"client_id\": \"%s\", \"client_secret\": \"%s\", \"refresh_token\": \"%s\"}"; + String payload = String.format(payloadTemplate, "refresh_token", clientId, clientSecret, refreshToken); + HttpEntity entity = new HttpEntity<>(payload, headers); + + Oauth2Config oauth2Config = confluenceSourceConfig.getAuthenticationConfig().getOauth2Config(); + try { + ResponseEntity responseEntity = restTemplate.postForEntity(TOKEN_LOCATION, entity, Map.class); + Map oauthClientResponse = responseEntity.getBody(); + this.accessToken = (String) oauthClientResponse.get(ACCESS_TOKEN); + this.refreshToken = (String) oauthClientResponse.get(REFRESH_TOKEN); + this.expiresInSeconds = (int) oauthClientResponse.get(EXPIRES_IN); + this.expireTime = Instant.now().plusSeconds(expiresInSeconds); + // updating config object's PluginConfigVariable so that it updates the underlying Secret store + oauth2Config.getAccessToken().setValue(this.accessToken); + oauth2Config.getRefreshToken().setValue(this.refreshToken); + log.info("Access Token and Refresh Token pair is now refreshed. Corresponding Secret store key updated."); + } catch (HttpClientErrorException ex) { + this.expireTime = Instant.ofEpochMilli(0); + this.expiresInSeconds = 0; + HttpStatus statusCode = ex.getStatusCode(); + log.error("Failed to renew access token. Status code: {}, Error Message: {}", + statusCode, ex.getMessage()); + if (statusCode == HttpStatus.FORBIDDEN || statusCode == HttpStatus.UNAUTHORIZED) { + log.info("Trying to refresh the secrets"); + // Refreshing the secrets. It should help if someone already renewed the tokens. + // Refreshing one of the secret refreshes the entire store so triggering refresh on just one + oauth2Config.getAccessToken().refresh(); + this.accessToken = (String) oauth2Config.getAccessToken().getValue(); + this.refreshToken = (String) oauth2Config.getRefreshToken().getValue(); + this.expireTime = Instant.now().plusSeconds(10); + } + throw new RuntimeException("Failed to renew access token message:" + ex.getMessage(), ex); + } + } + } + + @Override + public String getUrl() { + if (!StringUtils.hasLength(url)) { + synchronized (cloudIdFetchLock) { + if (!StringUtils.hasLength(url)) { + initCredentials(); + } + } + } + return url; + } + + /** + * Method for getting Jira url based on auth type. + */ + @Override + public void initCredentials() { + //For OAuth based flow, we use a different Jira url + this.cloudId = getJiraAccountCloudId(); + this.url = OAuth2_URL + this.cloudId + SLASH; + } +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/AddressValidation.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/AddressValidation.java new file mode 100644 index 0000000000..126ac3ec95 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/AddressValidation.java @@ -0,0 +1,60 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.utils; + +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import org.opensearch.dataprepper.plugins.source.confluence.exception.BadRequestException; + +import java.net.InetAddress; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.UnknownHostException; + + +/** + * This is the AddressValidation Class. + */ + +@Slf4j +public class AddressValidation { + + public static final String INVALID_URL = "URL is not valid "; + + /** + * Method for getInetAddress. + * + * @param url input parameter. + */ + public static InetAddress getInetAddress(String url) { + try { + return InetAddress.getByName(new URL(url).getHost()); + } catch (UnknownHostException | MalformedURLException e) { + log.error(INVALID_URL, e); + throw new BadRequestException(e.getMessage(), e); + } + } + + /** + * Validates the InetAddress and throws if the address is any of the following: 1. Link Local + * Address 2. Loopback + * Address 3. Multicast Address 4. Any Local Address 5. Site Local Address + * + * @param address the {@link InetAddress} to validate. + * @throws BadRequestException if the address is invalid. + */ + public static void validateInetAddress(@NonNull final InetAddress address) { + if (address.isMulticastAddress() || address.isAnyLocalAddress() || address.isLinkLocalAddress() + || address.isSiteLocalAddress() || address.isLoopbackAddress()) { + throw new BadRequestException(INVALID_URL); + } + } +} \ No newline at end of file diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/ConfluenceConfigHelper.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/ConfluenceConfigHelper.java new file mode 100644 index 0000000000..fa31c16955 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/ConfluenceConfigHelper.java @@ -0,0 +1,120 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.utils; + + +import lombok.extern.slf4j.Slf4j; +import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceSourceConfig; + +import java.util.ArrayList; +import java.util.List; + +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.BASIC; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.OAUTH2; + +/** + * The type Jira configuration. + */ +@Slf4j +public class ConfluenceConfigHelper { + /** + * Get Issue Status Filter from repository configuration. + * + * @return List Issue Status Filter. + */ + public static List getIssueStatusIncludeFilter(ConfluenceSourceConfig repositoryConfiguration) { + if (repositoryConfiguration.getFilterConfig() == null || repositoryConfiguration.getFilterConfig().getStatusConfig() == null) { + return new ArrayList<>(); + } + return repositoryConfiguration.getFilterConfig().getStatusConfig().getInclude(); + } + + public static List getIssueStatusExcludeFilter(ConfluenceSourceConfig repositoryConfiguration) { + if (repositoryConfiguration.getFilterConfig() == null || repositoryConfiguration.getFilterConfig().getStatusConfig() == null) { + return new ArrayList<>(); + } + return repositoryConfiguration.getFilterConfig().getStatusConfig().getExclude(); + } + + /** + * Get Issue Types Filter from repository configuration. + * + * @return List Issue Type Filter. + */ + public static List getIssueTypeIncludeFilter(ConfluenceSourceConfig repositoryConfiguration) { + if (repositoryConfiguration.getFilterConfig() == null || repositoryConfiguration.getFilterConfig().getIssueTypeConfig() == null) { + return new ArrayList<>(); + } + return repositoryConfiguration.getFilterConfig().getIssueTypeConfig().getInclude(); + } + + public static List getIssueTypeExcludeFilter(ConfluenceSourceConfig repositoryConfiguration) { + if (repositoryConfiguration.getFilterConfig() == null || repositoryConfiguration.getFilterConfig().getIssueTypeConfig() == null) { + return new ArrayList<>(); + } + return repositoryConfiguration.getFilterConfig().getIssueTypeConfig().getExclude(); + } + + /** + * Get Project Filter Types from repository configuration. + * public static final String ST = "status"; + * + * @return List Project Filter. + */ + public static List getProjectNameIncludeFilter(ConfluenceSourceConfig repositoryConfiguration) { + if (repositoryConfiguration.getFilterConfig() == null || + repositoryConfiguration.getFilterConfig().getProjectConfig() == null || + repositoryConfiguration.getFilterConfig().getProjectConfig().getNameConfig() == null) { + return new ArrayList<>(); + } + return repositoryConfiguration.getFilterConfig().getProjectConfig().getNameConfig().getInclude(); + } + + public static List getProjectNameExcludeFilter(ConfluenceSourceConfig repositoryConfiguration) { + if (repositoryConfiguration.getFilterConfig() == null || + repositoryConfiguration.getFilterConfig().getProjectConfig() == null || + repositoryConfiguration.getFilterConfig().getProjectConfig().getNameConfig() == null) { + return new ArrayList<>(); + } + return repositoryConfiguration.getFilterConfig().getProjectConfig().getNameConfig().getExclude(); + } + + + public static boolean validateConfig(ConfluenceSourceConfig config) { + if (config.getAccountUrl() == null) { + throw new RuntimeException("Account URL is missing."); + } + //At least one of the AuthType should be present + if (config.getAuthType() == null) { + throw new RuntimeException("Authentication Type is missing."); + } + String authType = config.getAuthType(); + if (!OAUTH2.equals(authType) && !BASIC.equals(authType)) { + throw new RuntimeException("Invalid AuthType is given"); + } + + if (BASIC.equals(authType)) { + if (config.getAuthenticationConfig().getBasicConfig().getUsername() == null || config.getAuthenticationConfig().getBasicConfig().getPassword() == null) { + throw new RuntimeException("Jira ID or Credential are required for Basic AuthType"); + } + } + + if (OAUTH2.equals(authType)) { + if (config.getAuthenticationConfig().getOauth2Config().getAccessToken() == null || config.getAuthenticationConfig().getOauth2Config().getRefreshToken() == null) { + throw new RuntimeException("Access Token or Refresh Token are required for OAuth2 AuthType"); + } + } + + AddressValidation.validateInetAddress(AddressValidation + .getInetAddress(config.getAccountUrl())); + return true; + } +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/ConfluenceContentType.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/ConfluenceContentType.java new file mode 100644 index 0000000000..06ec12c652 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/ConfluenceContentType.java @@ -0,0 +1,26 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.utils; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +@AllArgsConstructor +public enum ConfluenceContentType { + PROJECT("PROJECT"), + ISSUE("ISSUE"), + COMMENT("COMMENT"), + ATTACHMENT("ATTACHMENT"), + WORKLOG("WORKLOG"); + + @Getter + private final String type; +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/Constants.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/Constants.java new file mode 100644 index 0000000000..dafa4c3f90 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/Constants.java @@ -0,0 +1,34 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.utils; + +/** + * The type Constants. + */ +public class Constants { + + public static final int RETRY_ATTEMPT = 6; + + public static final String KEY = "key"; + public static final String NAME = "name"; + public static final String PROJECT = "project"; + public static final String OAUTH2 = "OAuth2"; + public static final String _PROJECT = "project-"; + public static final String _ISSUE = "ISSUE-"; + public static final String UPDATED = "updated"; + public static final String PROJECT_KEY = "j_project_key"; + public static final String PROJECT_NAME = "j_project_name"; + public static final String ISSUE_KEY = "j_issue_key"; + public static final String CREATED = "created"; + public static final String BASIC = "Basic"; + public static final String LIVE = "live"; + public static final String PLUGIN_NAME = "confluence"; +} \ No newline at end of file diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/JqlConstants.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/JqlConstants.java new file mode 100644 index 0000000000..7dfe523df2 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/JqlConstants.java @@ -0,0 +1,30 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.utils; + +public class JqlConstants { + public static final String GREATER_THAN_EQUALS = ">="; + public static final String CLOSING_ROUND_BRACKET = ")"; + + public static final String SLASH = "/"; + public static final String PROJECT_IN = " AND project in ("; + public static final String STATUS_IN = " AND status in ("; + public static final String PROJECT_NOT_IN = " AND project not in ("; + public static final String STATUS_NOT_IN = " AND status not in ("; + public static final String DELIMITER = "\",\""; + public static final String PREFIX = "\""; + public static final String SUFFIX = "\""; + public static final String ISSUE_TYPE_IN = " AND issueType in ("; + public static final String ISSUE_TYPE_NOT_IN = " AND issueType not in ("; + public static final String JQL_FIELD = "jql"; + public static final String EXPAND_FIELD = "expand"; + public static final String EXPAND_VALUE = "all"; +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceClientTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceClientTest.java new file mode 100644 index 0000000000..a75e6dfbc2 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceClientTest.java @@ -0,0 +1,149 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSet; +import org.opensearch.dataprepper.model.buffer.Buffer; +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.record.Record; +import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceClient; +import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceIterator; +import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceService; +import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceSourceConfig; +import org.opensearch.dataprepper.plugins.source.source_crawler.base.PluginExecutorServiceProvider; +import org.opensearch.dataprepper.plugins.source.source_crawler.coordination.state.SaasWorkerProgressState; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +public class ConfluenceClientTest { + + @Mock + private Buffer> buffer; + @Mock + private SaasWorkerProgressState saasWorkerProgressState; + @Mock + private AcknowledgementSet acknowledgementSet; + @Mock + private ConfluenceSourceConfig confluenceSourceConfig; + @Mock + private ConfluenceService confluenceService; + @Mock + private ConfluenceIterator confluenceIterator; + private final PluginExecutorServiceProvider executorServiceProvider = new PluginExecutorServiceProvider(); + + @Test + void testConstructor() { + ConfluenceClient confluenceClient = new ConfluenceClient(confluenceService, confluenceIterator, executorServiceProvider, confluenceSourceConfig); + confluenceClient.setLastPollTime(Instant.ofEpochSecond(1234L)); + assertNotNull(confluenceClient); + } + + @Test + void testListItems() { + ConfluenceClient confluenceClient = new ConfluenceClient(confluenceService, confluenceIterator, executorServiceProvider, confluenceSourceConfig); + assertNotNull(confluenceClient.listItems()); + } + + + @Test + void testExecutePartition() throws Exception { + ConfluenceClient confluenceClient = new ConfluenceClient(confluenceService, confluenceIterator, executorServiceProvider, confluenceSourceConfig); + Map keyAttributes = new HashMap<>(); + keyAttributes.put("project", "test"); + when(saasWorkerProgressState.getKeyAttributes()).thenReturn(keyAttributes); + List itemIds = new ArrayList<>(); + itemIds.add(null); + itemIds.add("ID2"); + itemIds.add("ID3"); + when(saasWorkerProgressState.getItemIds()).thenReturn(itemIds); + Instant exportStartTime = Instant.now(); + when(saasWorkerProgressState.getExportStartTime()).thenReturn(Instant.ofEpochSecond(exportStartTime.toEpochMilli())); + + when(confluenceService.getIssue(anyString())).thenReturn("{\"id\":\"ID1\",\"key\":\"TEST-1\"}"); + + ArgumentCaptor>> recordsCaptor = ArgumentCaptor.forClass((Class) Collection.class); + + confluenceClient.executePartition(saasWorkerProgressState, buffer, acknowledgementSet); + + verify(buffer).writeAll(recordsCaptor.capture(), anyInt()); + Collection> capturedRecords = recordsCaptor.getValue(); + assertFalse(capturedRecords.isEmpty()); + for (Record record : capturedRecords) { + assertNotNull(record.getData()); + } + } + + @Test + void testExecutePartitionError() throws Exception { + ConfluenceClient confluenceClient = new ConfluenceClient(confluenceService, confluenceIterator, executorServiceProvider, confluenceSourceConfig); + Map keyAttributes = new HashMap<>(); + keyAttributes.put("project", "test"); + when(saasWorkerProgressState.getKeyAttributes()).thenReturn(keyAttributes); + List itemIds = List.of("ID1", "ID2", "ID3", "ID4"); + when(saasWorkerProgressState.getItemIds()).thenReturn(itemIds); + Instant exportStartTime = Instant.now(); + when(saasWorkerProgressState.getExportStartTime()).thenReturn(Instant.ofEpochSecond(exportStartTime.toEpochMilli())); + + when(confluenceService.getIssue(anyString())).thenReturn("{\"id\":\"ID1\",\"key\":\"TEST-1\"}"); + + + ObjectMapper mockObjectMapper = mock(ObjectMapper.class); + when(mockObjectMapper.readValue(any(String.class), any(TypeReference.class))).thenThrow(new JsonProcessingException("test") { + }); + confluenceClient.injectObjectMapper(mockObjectMapper); + + assertThrows(RuntimeException.class, () -> confluenceClient.executePartition(saasWorkerProgressState, buffer, acknowledgementSet)); + } + + @Test + void bufferWriteRuntimeTest() throws Exception { + ConfluenceClient confluenceClient = new ConfluenceClient(confluenceService, confluenceIterator, executorServiceProvider, confluenceSourceConfig); + Map keyAttributes = new HashMap<>(); + keyAttributes.put("project", "test"); + when(saasWorkerProgressState.getKeyAttributes()).thenReturn(keyAttributes); + List itemIds = List.of("ID1", "ID2", "ID3", "ID4"); + when(saasWorkerProgressState.getItemIds()).thenReturn(itemIds); + Instant exportStartTime = Instant.now(); + when(saasWorkerProgressState.getExportStartTime()).thenReturn(Instant.ofEpochSecond(exportStartTime.toEpochMilli())); + + when(confluenceService.getIssue(anyString())).thenReturn("{\"id\":\"ID1\",\"key\":\"TEST-1\"}"); + + ArgumentCaptor>> recordsCaptor = ArgumentCaptor.forClass((Class) Collection.class); + + doThrow(new RuntimeException()).when(buffer).writeAll(recordsCaptor.capture(), anyInt()); + assertThrows(RuntimeException.class, () -> confluenceClient.executePartition(saasWorkerProgressState, buffer, acknowledgementSet)); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceConfigHelperTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceConfigHelperTest.java new file mode 100644 index 0000000000..4bddc9d839 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceConfigHelperTest.java @@ -0,0 +1,173 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.model.plugin.PluginConfigVariable; +import org.opensearch.dataprepper.plugins.source.confluence.configuration.AuthenticationConfig; +import org.opensearch.dataprepper.plugins.source.confluence.configuration.BasicConfig; +import org.opensearch.dataprepper.plugins.source.confluence.configuration.FilterConfig; +import org.opensearch.dataprepper.plugins.source.confluence.configuration.IssueTypeConfig; +import org.opensearch.dataprepper.plugins.source.confluence.configuration.NameConfig; +import org.opensearch.dataprepper.plugins.source.confluence.configuration.Oauth2Config; +import org.opensearch.dataprepper.plugins.source.confluence.configuration.ProjectConfig; +import org.opensearch.dataprepper.plugins.source.confluence.configuration.StatusConfig; +import org.opensearch.dataprepper.plugins.source.confluence.utils.ConfluenceConfigHelper; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.when; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.BASIC; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.OAUTH2; + +@ExtendWith(MockitoExtension.class) +public class ConfluenceConfigHelperTest { + + @Mock + ConfluenceSourceConfig confluenceSourceConfig; + + @Mock + FilterConfig filterConfig; + + @Mock + StatusConfig statusConfig; + + @Mock + IssueTypeConfig issueTypeConfig; + + @Mock + ProjectConfig projectConfig; + + @Mock + NameConfig nameConfig; + + @Mock + AuthenticationConfig authenticationConfig; + + @Mock + BasicConfig basicConfig; + + @Mock + Oauth2Config oauth2Config; + + @Mock + PluginConfigVariable accessTokenPluginConfigVariable; + + @Mock + PluginConfigVariable refreshTokenPluginConfigVariable; + + @Test + void testInitialization() { + ConfluenceConfigHelper confluenceConfigHelper = new ConfluenceConfigHelper(); + assertNotNull(confluenceConfigHelper); + } + + @Test + void testGetIssueStatusFilter() { + when(confluenceSourceConfig.getFilterConfig()).thenReturn(filterConfig); + when(filterConfig.getStatusConfig()).thenReturn(statusConfig); + assertTrue(ConfluenceConfigHelper.getIssueStatusIncludeFilter(confluenceSourceConfig).isEmpty()); + assertTrue(ConfluenceConfigHelper.getIssueStatusExcludeFilter(confluenceSourceConfig).isEmpty()); + List issueStatusFilter = List.of("Done", "In Progress"); + List issueStatusExcludeFilter = List.of("Done2", "In Progress2"); + when(statusConfig.getInclude()).thenReturn(issueStatusFilter); + when(statusConfig.getExclude()).thenReturn(issueStatusExcludeFilter); + assertEquals(issueStatusFilter, ConfluenceConfigHelper.getIssueStatusIncludeFilter(confluenceSourceConfig)); + assertEquals(issueStatusExcludeFilter, ConfluenceConfigHelper.getIssueStatusExcludeFilter(confluenceSourceConfig)); + } + + @Test + void testGetIssueTypeFilter() { + when(confluenceSourceConfig.getFilterConfig()).thenReturn(filterConfig); + when(filterConfig.getIssueTypeConfig()).thenReturn(issueTypeConfig); + assertTrue(ConfluenceConfigHelper.getIssueTypeIncludeFilter(confluenceSourceConfig).isEmpty()); + assertTrue(ConfluenceConfigHelper.getIssueTypeExcludeFilter(confluenceSourceConfig).isEmpty()); + List issueTypeFilter = List.of("Bug", "Story"); + List issueTypeExcludeFilter = List.of("Bug2", "Story2"); + when(issueTypeConfig.getInclude()).thenReturn(issueTypeFilter); + when(issueTypeConfig.getExclude()).thenReturn(issueTypeExcludeFilter); + assertEquals(issueTypeFilter, ConfluenceConfigHelper.getIssueTypeIncludeFilter(confluenceSourceConfig)); + assertEquals(issueTypeExcludeFilter, ConfluenceConfigHelper.getIssueTypeExcludeFilter(confluenceSourceConfig)); + } + + @Test + void testGetProjectNameFilter() { + when(confluenceSourceConfig.getFilterConfig()).thenReturn(filterConfig); + when(filterConfig.getProjectConfig()).thenReturn(projectConfig); + when(projectConfig.getNameConfig()).thenReturn(nameConfig); + assertTrue(ConfluenceConfigHelper.getProjectNameIncludeFilter(confluenceSourceConfig).isEmpty()); + assertTrue(ConfluenceConfigHelper.getProjectNameExcludeFilter(confluenceSourceConfig).isEmpty()); + List projectNameFilter = List.of("TEST", "TEST2"); + List projectNameExcludeFilter = List.of("TEST3", "TEST4"); + when(nameConfig.getInclude()).thenReturn(projectNameFilter); + when(nameConfig.getExclude()).thenReturn(projectNameExcludeFilter); + assertEquals(projectNameFilter, ConfluenceConfigHelper.getProjectNameIncludeFilter(confluenceSourceConfig)); + assertEquals(projectNameExcludeFilter, ConfluenceConfigHelper.getProjectNameExcludeFilter(confluenceSourceConfig)); + } + + + @Test + void testValidateConfig() { + assertThrows(RuntimeException.class, () -> ConfluenceConfigHelper.validateConfig(confluenceSourceConfig)); + + when(confluenceSourceConfig.getAccountUrl()).thenReturn("https://test.com"); + assertThrows(RuntimeException.class, () -> ConfluenceConfigHelper.validateConfig(confluenceSourceConfig)); + + when(confluenceSourceConfig.getAuthType()).thenReturn("fakeType"); + assertThrows(RuntimeException.class, () -> ConfluenceConfigHelper.validateConfig(confluenceSourceConfig)); + } + + @Test + void testValidateConfigBasic() { + when(confluenceSourceConfig.getAccountUrl()).thenReturn("https://test.com"); + when(confluenceSourceConfig.getAuthType()).thenReturn(BASIC); + when(confluenceSourceConfig.getAuthenticationConfig()).thenReturn(authenticationConfig); + when(authenticationConfig.getBasicConfig()).thenReturn(basicConfig); + assertThrows(RuntimeException.class, () -> ConfluenceConfigHelper.validateConfig(confluenceSourceConfig)); + + when(basicConfig.getUsername()).thenReturn("id"); + assertThrows(RuntimeException.class, () -> ConfluenceConfigHelper.validateConfig(confluenceSourceConfig)); + + when(basicConfig.getPassword()).thenReturn("credential"); + when(basicConfig.getUsername()).thenReturn(null); + assertThrows(RuntimeException.class, () -> ConfluenceConfigHelper.validateConfig(confluenceSourceConfig)); + + when(basicConfig.getUsername()).thenReturn("id"); + assertDoesNotThrow(() -> ConfluenceConfigHelper.validateConfig(confluenceSourceConfig)); + } + + @Test + void testValidateConfigOauth2() { + when(confluenceSourceConfig.getAccountUrl()).thenReturn("https://test.com"); + when(confluenceSourceConfig.getAuthType()).thenReturn(OAUTH2); + when(confluenceSourceConfig.getAuthenticationConfig()).thenReturn(authenticationConfig); + when(authenticationConfig.getOauth2Config()).thenReturn(oauth2Config); + assertThrows(RuntimeException.class, () -> ConfluenceConfigHelper.validateConfig(confluenceSourceConfig)); + + when(oauth2Config.getAccessToken()).thenReturn(accessTokenPluginConfigVariable); + assertThrows(RuntimeException.class, () -> ConfluenceConfigHelper.validateConfig(confluenceSourceConfig)); + + when(authenticationConfig.getOauth2Config().getRefreshToken()).thenReturn(refreshTokenPluginConfigVariable); + when(oauth2Config.getAccessToken()).thenReturn(null); + assertThrows(RuntimeException.class, () -> ConfluenceConfigHelper.validateConfig(confluenceSourceConfig)); + + when(oauth2Config.getAccessToken()).thenReturn(accessTokenPluginConfigVariable); + assertDoesNotThrow(() -> ConfluenceConfigHelper.validateConfig(confluenceSourceConfig)); + } +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceItemInfoTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceItemInfoTest.java new file mode 100644 index 0000000000..f10249b686 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceItemInfoTest.java @@ -0,0 +1,109 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceItemInfo; +import org.opensearch.dataprepper.plugins.source.confluence.utils.Constants; + +import java.time.Instant; +import java.util.Map; +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +public class ConfluenceItemInfoTest { + private String project; + private String issueType; + private String id; + private String itemId; + private Instant eventTime; + + @Mock + private Map metadata; + + @Mock + private Map newMetadata; + + @Mock + private ConfluenceItemInfo confluenceItemInfo; + + @BeforeEach + void setUP() { + issueType = "TestIssue"; + id = UUID.randomUUID().toString(); + project = "TestProject"; + itemId = UUID.randomUUID().toString(); + eventTime = Instant.ofEpochSecond(0); + confluenceItemInfo = new ConfluenceItemInfo(id, itemId, project, issueType, metadata, eventTime); + } + + @Test + void testGetters() { + assertEquals(confluenceItemInfo.getItemId(), itemId); + assertEquals(confluenceItemInfo.getId(), id); + assertEquals(confluenceItemInfo.getProject(), project); + assertEquals(confluenceItemInfo.getIssueType(), issueType); + assertEquals(confluenceItemInfo.getMetadata(), metadata); + assertEquals(confluenceItemInfo.getEventTime(), eventTime); + } + + @Test + void testGetKeyAttributes() { + assertInstanceOf(Map.class, confluenceItemInfo.getKeyAttributes()); + } + + @Test + void testSetter() { + confluenceItemInfo.setEventTime(Instant.now()); + assertNotEquals(confluenceItemInfo.getEventTime(), eventTime); + confluenceItemInfo.setItemId("newItemID"); + assertNotEquals(confluenceItemInfo.getItemId(), itemId); + confluenceItemInfo.setId("newID"); + assertNotEquals(confluenceItemInfo.getId(), id); + confluenceItemInfo.setProject("newProject"); + assertNotEquals(confluenceItemInfo.getProject(), project); + confluenceItemInfo.setMetadata(newMetadata); + assertNotEquals(confluenceItemInfo.getMetadata(), metadata); + confluenceItemInfo.setIssueType("newIssueType"); + assertNotEquals(confluenceItemInfo.getIssueType(), issueType); + + } + + @Test + void testGetPartitionKey() { + String partitionKey = confluenceItemInfo.getPartitionKey(); + assertTrue(partitionKey.contains(project)); + assertTrue(partitionKey.contains(issueType)); + } + + + @Test + void testGetLastModifiedAt() { + when(metadata.get(Constants.UPDATED)).thenReturn("5"); + when(metadata.get(Constants.CREATED)).thenReturn("0"); + assertEquals(Instant.ofEpochMilli(5), confluenceItemInfo.getLastModifiedAt()); + + when(metadata.get(Constants.UPDATED)).thenReturn("5"); + when(metadata.get(Constants.CREATED)).thenReturn("7"); + assertEquals(Instant.ofEpochMilli(7), confluenceItemInfo.getLastModifiedAt()); + } + +} \ No newline at end of file diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceIteratorTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceIteratorTest.java new file mode 100644 index 0000000000..ac64972c72 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceIteratorTest.java @@ -0,0 +1,201 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.plugins.source.confluence.models.IssueBean; +import org.opensearch.dataprepper.plugins.source.confluence.models.SearchResults; +import org.opensearch.dataprepper.plugins.source.confluence.rest.ConfluenceRestClient; +import org.opensearch.dataprepper.plugins.source.source_crawler.base.PluginExecutorServiceProvider; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.CREATED; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.KEY; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.UPDATED; + +@ExtendWith(MockitoExtension.class) +public class ConfluenceIteratorTest { + + @Mock + private SearchResults mockSearchResults; + @Mock + private ConfluenceRestClient confluenceRestClient; + private ConfluenceService confluenceService; + @Mock + private ConfluenceSourceConfig confluenceSourceConfig; + private ConfluenceIterator confluenceIterator; + private final PluginExecutorServiceProvider executorServiceProvider = new PluginExecutorServiceProvider(); + + @BeforeEach + void setUp() { + confluenceService = spy(new ConfluenceService(confluenceSourceConfig, confluenceRestClient)); + } + + public ConfluenceIterator createObjectUnderTest() { + return new ConfluenceIterator(confluenceService, executorServiceProvider, confluenceSourceConfig); + } + + @Test + void testInitialization() { + confluenceIterator = createObjectUnderTest(); + assertNotNull(confluenceIterator); + confluenceIterator.initialize(Instant.ofEpochSecond(0)); + when(mockSearchResults.getIssues()).thenReturn(new ArrayList<>()); + when(mockSearchResults.getTotal()).thenReturn(0); + doReturn(mockSearchResults).when(confluenceRestClient).getAllIssues(any(StringBuilder.class), anyInt(), any(ConfluenceSourceConfig.class)); + assertFalse(confluenceIterator.hasNext()); + } + + @Test + void sleepInterruptionTest() { + confluenceIterator = createObjectUnderTest(); + confluenceIterator.initialize(Instant.ofEpochSecond(0)); + + Thread testThread = new Thread(() -> { + assertThrows(InterruptedException.class, () -> { + try { + confluenceIterator.hasNext(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + }); + + testThread.start(); + testThread.interrupt(); + } + + @Test + void testItemInfoQueueNotEmpty() { + confluenceIterator = createObjectUnderTest(); + List mockIssues = new ArrayList<>(); + IssueBean issue1 = createIssueBean(false); + mockIssues.add(issue1); + when(mockSearchResults.getIssues()).thenReturn(mockIssues); + when(mockSearchResults.getTotal()).thenReturn(0); + doReturn(mockSearchResults).when(confluenceRestClient).getAllIssues(any(StringBuilder.class), anyInt(), any(ConfluenceSourceConfig.class)); + + confluenceIterator.initialize(Instant.ofEpochSecond(0)); + confluenceIterator.setCrawlerQWaitTimeMillis(1); + assertTrue(confluenceIterator.hasNext()); + assertNotNull(confluenceIterator.next()); + } + + @Test + void testStartCrawlerThreads() { + confluenceIterator = createObjectUnderTest(); + confluenceIterator.initialize(Instant.ofEpochSecond(0)); + confluenceIterator.hasNext(); + confluenceIterator.hasNext(); + assertEquals(1, confluenceIterator.showFutureList().size()); + } + + @Test + void testFuturesCompleted() throws InterruptedException { + confluenceIterator = createObjectUnderTest(); + List mockIssues = new ArrayList<>(); + IssueBean issue1 = createIssueBean(false); + mockIssues.add(issue1); + IssueBean issue2 = createIssueBean(false); + mockIssues.add(issue2); + IssueBean issue3 = createIssueBean(false); + mockIssues.add(issue3); + when(mockSearchResults.getIssues()).thenReturn(mockIssues); + when(mockSearchResults.getTotal()).thenReturn(0); + doReturn(mockSearchResults).when(confluenceRestClient).getAllIssues(any(StringBuilder.class), anyInt(), any(ConfluenceSourceConfig.class)); + + confluenceIterator.initialize(Instant.ofEpochSecond(0)); + confluenceIterator.setCrawlerQWaitTimeMillis(1); + confluenceIterator.hasNext(); + + Thread.sleep(1); + confluenceIterator.showFutureList().forEach(future -> assertTrue(future.isDone())); + assertEquals(confluenceIterator.showItemInfoQueue().size(), mockIssues.size()); + } + + @Test + void testItemInfoQueueEmpty() { + confluenceIterator = createObjectUnderTest(); + List mockIssues = new ArrayList<>(); + when(mockSearchResults.getIssues()).thenReturn(mockIssues); + when(mockSearchResults.getTotal()).thenReturn(0); + doReturn(mockSearchResults).when(confluenceRestClient).getAllIssues(any(StringBuilder.class), anyInt(), any(ConfluenceSourceConfig.class)); + + confluenceIterator.initialize(Instant.ofEpochSecond(0)); + confluenceIterator.setCrawlerQWaitTimeMillis(1); + assertFalse(confluenceIterator.hasNext()); + assertThrows(NoSuchElementException.class, () -> confluenceIterator.next()); + } + + + private IssueBean createIssueBean(boolean nullFields) { + IssueBean issue1 = new IssueBean(); + issue1.setId(UUID.randomUUID().toString()); + issue1.setKey("issue_1_key"); + issue1.setSelf("https://example.com/rest/api/2/issue/123"); + issue1.setExpand("operations,versionedRepresentations,editmeta"); + + Map fieldMap = new HashMap<>(); + if (!nullFields) { + fieldMap.put(CREATED, Instant.now()); + fieldMap.put(UPDATED, Instant.now()); + } else { + fieldMap.put(CREATED, 0); + fieldMap.put(UPDATED, 0); + } + + Map issueTypeMap = new HashMap<>(); + issueTypeMap.put("name", "Task"); + issueTypeMap.put("self", "https://example.com/rest/api/2/issuetype/1"); + issueTypeMap.put("id", "1"); + fieldMap.put("issuetype", issueTypeMap); + + Map projectMap = new HashMap<>(); + if (!nullFields) { + projectMap.put("name", "project name test"); + projectMap.put(KEY, "TEST"); + } + fieldMap.put("project", projectMap); + + Map priorityMap = new HashMap<>(); + priorityMap.put("name", "Medium"); + fieldMap.put("priority", priorityMap); + + Map statusMap = new HashMap<>(); + statusMap.put("name", "In Progress"); + fieldMap.put("status", statusMap); + + issue1.setFields(fieldMap); + + return issue1; + } +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceServiceTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceServiceTest.java new file mode 100644 index 0000000000..9859ea3051 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceServiceTest.java @@ -0,0 +1,325 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.model.plugin.PluginConfigVariable; +import org.opensearch.dataprepper.plugins.source.confluence.configuration.Oauth2Config; +import org.opensearch.dataprepper.plugins.source.confluence.exception.BadRequestException; +import org.opensearch.dataprepper.plugins.source.confluence.models.IssueBean; +import org.opensearch.dataprepper.plugins.source.confluence.models.SearchResults; +import org.opensearch.dataprepper.plugins.source.confluence.rest.ConfluenceRestClient; +import org.opensearch.dataprepper.plugins.source.confluence.utils.MockPluginConfigVariableImpl; +import org.opensearch.dataprepper.plugins.source.source_crawler.base.PluginExecutorServiceProvider; +import org.opensearch.dataprepper.plugins.source.source_crawler.model.ItemInfo; +import org.opensearch.dataprepper.test.helper.ReflectivelySetField; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.InputStream; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.UUID; +import java.util.concurrent.ConcurrentLinkedQueue; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; +import static org.opensearch.dataprepper.plugins.source.confluence.rest.auth.ConfluenceOauthConfig.ACCESSIBLE_RESOURCES; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.BASIC; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.CREATED; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.KEY; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.NAME; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.OAUTH2; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.PROJECT; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.UPDATED; + + +/** + * The type Jira service. + */ +@ExtendWith(MockitoExtension.class) +public class ConfluenceServiceTest { + + private static final Logger log = LoggerFactory.getLogger(ConfluenceServiceTest.class); + @Mock + private ConfluenceRestClient confluenceRestClient; + private final PluginExecutorServiceProvider executorServiceProvider = new PluginExecutorServiceProvider(); + + private static InputStream getResourceAsStream(String resourceName) { + InputStream inputStream = Thread.currentThread().getContextClassLoader().getResourceAsStream(resourceName); + if (inputStream == null) { + inputStream = ConfluenceServiceTest.class.getResourceAsStream("/" + resourceName); + } + return inputStream; + } + + public static ConfluenceSourceConfig createJiraConfigurationFromYaml(String fileName) { + ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory()); + try (InputStream inputStream = getResourceAsStream(fileName)) { + ConfluenceSourceConfig confluenceSourceConfig = objectMapper.readValue(inputStream, ConfluenceSourceConfig.class); + Oauth2Config oauth2Config = confluenceSourceConfig.getAuthenticationConfig().getOauth2Config(); + if (oauth2Config != null) { + ReflectivelySetField.setField(Oauth2Config.class, oauth2Config, "accessToken", + new MockPluginConfigVariableImpl("mockAccessToken")); + ReflectivelySetField.setField(Oauth2Config.class, oauth2Config, "refreshToken", + new MockPluginConfigVariableImpl("mockRefreshToken")); + } + return confluenceSourceConfig; + } catch (IOException ex) { + log.error("Failed to parse pipeline Yaml", ex); + } catch (Exception e) { + throw new RuntimeException(e); + } + return null; + } + + public static ConfluenceSourceConfig createJiraConfiguration(String auth_type, + List issueType, + List issueStatus, + List projectKey) throws JsonProcessingException { + PluginConfigVariable pcvAccessToken = null; + PluginConfigVariable pcvRefreshToken = null; + ObjectMapper objectMapper = new ObjectMapper(); + Map authenticationMap = new HashMap<>(); + Map basicMap = new HashMap<>(); + Map oauth2Map = new HashMap<>(); + if (auth_type.equals(BASIC)) { + basicMap.put("username", "test_username"); + basicMap.put("password", "test_password"); + authenticationMap.put("basic", basicMap); + } else if (auth_type.equals(OAUTH2)) { + oauth2Map.put("client_id", "test-client-id"); + oauth2Map.put("client_secret", "test-client-secret"); + pcvAccessToken = new MockPluginConfigVariableImpl("test-access-token"); + pcvRefreshToken = new MockPluginConfigVariableImpl("test-refresh-token"); + authenticationMap.put("oauth2", oauth2Map); + } + + Map jiraSourceConfigMap = new HashMap<>(); + List hosts = new ArrayList<>(); + hosts.add(ACCESSIBLE_RESOURCES); + + Map filterMap = new HashMap<>(); + Map projectMap = new HashMap<>(); + Map issueTypeMap = new HashMap<>(); + Map statusMap = new HashMap<>(); + + issueTypeMap.put("include", issueType); + filterMap.put("issue_type", issueTypeMap); + + statusMap.put("include", issueStatus); + filterMap.put("status", statusMap); + + Map nameMap = new HashMap<>(); + nameMap.put("include", projectKey); + projectMap.put("key", nameMap); + filterMap.put("project", projectMap); + + + jiraSourceConfigMap.put("hosts", hosts); + jiraSourceConfigMap.put("authentication", authenticationMap); + jiraSourceConfigMap.put("filter", filterMap); + + String jiraSourceConfigJsonString = objectMapper.writeValueAsString(jiraSourceConfigMap); + ConfluenceSourceConfig confluenceSourceConfig = objectMapper.readValue(jiraSourceConfigJsonString, ConfluenceSourceConfig.class); + if (confluenceSourceConfig.getAuthenticationConfig().getOauth2Config() != null && pcvAccessToken != null) { + try { + ReflectivelySetField.setField(Oauth2Config.class, + confluenceSourceConfig.getAuthenticationConfig().getOauth2Config(), "accessToken", pcvAccessToken); + ReflectivelySetField.setField(Oauth2Config.class, + confluenceSourceConfig.getAuthenticationConfig().getOauth2Config(), "refreshToken", pcvRefreshToken); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + return confluenceSourceConfig; + } + + @AfterEach + void tearDown() { + executorServiceProvider.terminateExecutor(); + } + + @Test + void testJiraServiceInitialization() throws JsonProcessingException { + List issueType = new ArrayList<>(); + List issueStatus = new ArrayList<>(); + List projectKey = new ArrayList<>(); + ConfluenceSourceConfig confluenceSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey); + ConfluenceService confluenceService = new ConfluenceService(confluenceSourceConfig, confluenceRestClient); + assertNotNull(confluenceService); + when(confluenceRestClient.getIssue(anyString())).thenReturn("test String"); + assertNotNull(confluenceService.getIssue("test Key")); + } + + @Test + public void testGetJiraEntities() throws JsonProcessingException { + List issueType = new ArrayList<>(); + List issueStatus = new ArrayList<>(); + List projectKey = new ArrayList<>(); + issueType.add("Task"); + issueStatus.add("Done"); + projectKey.add("KAN"); + ConfluenceSourceConfig confluenceSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey); + ConfluenceService confluenceService = spy(new ConfluenceService(confluenceSourceConfig, confluenceRestClient)); + List mockIssues = new ArrayList<>(); + IssueBean issue1 = createIssueBean(false, false); + mockIssues.add(issue1); + IssueBean issue2 = createIssueBean(true, false); + mockIssues.add(issue2); + IssueBean issue3 = createIssueBean(false, true); + mockIssues.add(issue3); + + SearchResults mockSearchResults = mock(SearchResults.class); + when(mockSearchResults.getIssues()).thenReturn(mockIssues); + when(mockSearchResults.getTotal()).thenReturn(mockIssues.size()); + + doReturn(mockSearchResults).when(confluenceRestClient).getAllIssues(any(StringBuilder.class), anyInt(), any(ConfluenceSourceConfig.class)); + + Instant timestamp = Instant.ofEpochSecond(0); + Queue itemInfoQueue = new ConcurrentLinkedQueue<>(); + confluenceService.getJiraEntities(confluenceSourceConfig, timestamp, itemInfoQueue); + assertEquals(mockIssues.size(), itemInfoQueue.size()); + } + + @Test + public void buildIssueItemInfoMultipleFutureThreads() throws JsonProcessingException { + List issueType = new ArrayList<>(); + List issueStatus = new ArrayList<>(); + List projectKey = new ArrayList<>(); + issueType.add("Task"); + ConfluenceSourceConfig confluenceSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey); + ConfluenceService confluenceService = spy(new ConfluenceService(confluenceSourceConfig, confluenceRestClient)); + List mockIssues = new ArrayList<>(); + for (int i = 0; i < 50; i++) { + IssueBean issue1 = createIssueBean(false, false); + mockIssues.add(issue1); + } + + SearchResults mockSearchResults = mock(SearchResults.class); + when(mockSearchResults.getIssues()).thenReturn(mockIssues); + when(mockSearchResults.getTotal()).thenReturn(100); + + doReturn(mockSearchResults).when(confluenceRestClient).getAllIssues(any(StringBuilder.class), anyInt(), any(ConfluenceSourceConfig.class)); + + Instant timestamp = Instant.ofEpochSecond(0); + Queue itemInfoQueue = new ConcurrentLinkedQueue<>(); + confluenceService.getJiraEntities(confluenceSourceConfig, timestamp, itemInfoQueue); + assertTrue(itemInfoQueue.size() >= 100); + } + + @Test + public void testBadProjectKeys() throws JsonProcessingException { + List issueType = new ArrayList<>(); + List issueStatus = new ArrayList<>(); + List projectKey = new ArrayList<>(); + issueType.add("Task"); + issueStatus.add("Done"); + projectKey.add("Bad Project Key"); + projectKey.add("A"); + projectKey.add("!@#$"); + projectKey.add("AAAAAAAAAAAAAA"); + + ConfluenceSourceConfig confluenceSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey); + ConfluenceService confluenceService = new ConfluenceService(confluenceSourceConfig, confluenceRestClient); + + Instant timestamp = Instant.ofEpochSecond(0); + Queue itemInfoQueue = new ConcurrentLinkedQueue<>(); + + assertThrows(BadRequestException.class, () -> confluenceService.getJiraEntities(confluenceSourceConfig, timestamp, itemInfoQueue)); + } + + @Test + public void testGetJiraEntitiesException() throws JsonProcessingException { + List issueType = new ArrayList<>(); + List issueStatus = new ArrayList<>(); + List projectKey = new ArrayList<>(); + issueType.add("Task"); + ConfluenceSourceConfig confluenceSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey); + ConfluenceService confluenceService = spy(new ConfluenceService(confluenceSourceConfig, confluenceRestClient)); + + doThrow(RuntimeException.class).when(confluenceRestClient).getAllIssues(any(StringBuilder.class), anyInt(), any(ConfluenceSourceConfig.class)); + + Instant timestamp = Instant.ofEpochSecond(0); + Queue itemInfoQueue = new ConcurrentLinkedQueue<>(); + + assertThrows(RuntimeException.class, () -> confluenceService.getJiraEntities(confluenceSourceConfig, timestamp, itemInfoQueue)); + } + + + private IssueBean createIssueBean(boolean nullFields, boolean createdNull) { + IssueBean issue1 = new IssueBean(); + issue1.setId(UUID.randomUUID().toString()); + issue1.setKey("issue_1_key"); + issue1.setSelf("https://example.com/rest/api/2/issue/123"); + issue1.setExpand("operations,versionedRepresentations,editmeta"); + + Map fieldMap = new HashMap<>(); + if (!nullFields) { + fieldMap.put(CREATED, "2024-07-06T21:12:23.437-0700"); + fieldMap.put(UPDATED, "2024-07-06T21:12:23.106-0700"); + } else { + fieldMap.put(CREATED, 0); + fieldMap.put(UPDATED, 0); + } + if (createdNull) { + fieldMap.put(CREATED, null); + } + + Map issueTypeMap = new HashMap<>(); + issueTypeMap.put("name", "Task"); + issueTypeMap.put("self", "https://example.com/rest/api/2/issuetype/1"); + issueTypeMap.put("id", "1"); + fieldMap.put("issuetype", issueTypeMap); + + Map projectMap = new HashMap<>(); + if (!nullFields) { + projectMap.put(NAME, "project name test"); + projectMap.put(KEY, "TEST"); + } + fieldMap.put(PROJECT, projectMap); + + Map priorityMap = new HashMap<>(); + priorityMap.put(NAME, "Medium"); + fieldMap.put("priority", priorityMap); + + Map statusMap = new HashMap<>(); + statusMap.put(NAME, "In Progress"); + fieldMap.put("statuses", statusMap); + + issue1.setFields(fieldMap); + + return issue1; + } + +} \ No newline at end of file diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceConfigTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceConfigTest.java new file mode 100644 index 0000000000..d3bbb7411b --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceConfigTest.java @@ -0,0 +1,137 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; +import org.opensearch.dataprepper.model.plugin.PluginConfigVariable; +import org.opensearch.dataprepper.plugins.source.confluence.configuration.Oauth2Config; +import org.opensearch.dataprepper.plugins.source.confluence.utils.MockPluginConfigVariableImpl; +import org.opensearch.dataprepper.test.helper.ReflectivelySetField; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.BASIC; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.OAUTH2; + +public class ConfluenceSourceConfigTest { + private ConfluenceSourceConfig confluenceSourceConfig; + private final PluginConfigVariable accessToken = new MockPluginConfigVariableImpl("access token test"); + private final PluginConfigVariable refreshToken = new MockPluginConfigVariableImpl("refresh token test"); + private final String clientId = "client id test"; + private final String clientSecret = "client secret test"; + private final String password = "test Jira Credential"; + private final String username = "test Jira Id"; + private final String accountUrl = "https://example.atlassian.net"; + private final List projectList = new ArrayList<>(); + private final List issueTypeList = new ArrayList<>(); + private final List statusList = new ArrayList<>(); + + private ConfluenceSourceConfig createJiraSourceConfig(String authtype, boolean hasToken) throws Exception { + PluginConfigVariable pcvAccessToken = null; + PluginConfigVariable pcvRefreshToken = null; + Map configMap = new HashMap<>(); + List hosts = new ArrayList<>(); + hosts.add(accountUrl); + + configMap.put("hosts", hosts); + + Map authenticationMap = new HashMap<>(); + Map basicMap = new HashMap<>(); + Map oauth2Map = new HashMap<>(); + if (authtype.equals(BASIC)) { + basicMap.put("username", username); + basicMap.put("password", password); + authenticationMap.put("basic", basicMap); + } else if (authtype.equals(OAUTH2)) { + if (hasToken) { + pcvRefreshToken = refreshToken; + pcvAccessToken = accessToken; + } else { + oauth2Map.put("refresh_token", null); + } + oauth2Map.put("client_id", clientId); + oauth2Map.put("client_secret", clientSecret); + authenticationMap.put("oauth2", oauth2Map); + } + + configMap.put("authentication", authenticationMap); + + projectList.add("project1"); + projectList.add("project2"); + + issueTypeList.add("issue type 1"); + issueTypeList.add("issue type 2"); + + statusList.add("status 1"); + statusList.add("status 2"); + + Map filterMap = new HashMap<>(); + Map projectMap = new HashMap<>(); + Map issueTypeMap = new HashMap<>(); + Map statusMap = new HashMap<>(); + + issueTypeMap.put("include", issueTypeList); + filterMap.put("issue_type", issueTypeMap); + + statusMap.put("include", statusList); + filterMap.put("status", statusMap); + + Map nameMap = new HashMap<>(); + nameMap.put("include", projectList); + projectMap.put("key", nameMap); + filterMap.put("project", projectMap); + + configMap.put("filter", filterMap); + + ObjectMapper objectMapper = new ObjectMapper(); + String jsonConfig = objectMapper.writeValueAsString(configMap); + ConfluenceSourceConfig config = objectMapper.readValue(jsonConfig, ConfluenceSourceConfig.class); + if (config.getAuthenticationConfig().getOauth2Config() != null && pcvAccessToken != null) { + ReflectivelySetField.setField(Oauth2Config.class, + config.getAuthenticationConfig().getOauth2Config(), "accessToken", pcvAccessToken); + ReflectivelySetField.setField(Oauth2Config.class, + config.getAuthenticationConfig().getOauth2Config(), "refreshToken", pcvRefreshToken); + } + return config; + } + + @Test + void testGetters() throws Exception { + confluenceSourceConfig = createJiraSourceConfig(BASIC, false); + assertEquals(confluenceSourceConfig.getFilterConfig().getIssueTypeConfig().getInclude(), issueTypeList); + assertEquals(confluenceSourceConfig.getFilterConfig().getProjectConfig().getNameConfig().getInclude(), projectList); + assertEquals(confluenceSourceConfig.getFilterConfig().getStatusConfig().getInclude(), statusList); + assertEquals(confluenceSourceConfig.getAccountUrl(), accountUrl); + assertEquals(confluenceSourceConfig.getAuthenticationConfig().getBasicConfig().getPassword(), password); + assertEquals(confluenceSourceConfig.getAuthenticationConfig().getBasicConfig().getUsername(), username); + } + + @Test + void testFetchGivenOauthAttributeWrongAuthType() throws Exception { + confluenceSourceConfig = createJiraSourceConfig(BASIC, true); + assertThrows(RuntimeException.class, () -> confluenceSourceConfig.getAuthenticationConfig().getOauth2Config().getAccessToken()); + } + + @Test + void testFetchGivenOauthAtrribute() throws Exception { + confluenceSourceConfig = createJiraSourceConfig(OAUTH2, true); + assertEquals(accessToken, confluenceSourceConfig.getAuthenticationConfig().getOauth2Config().getAccessToken()); + assertEquals(refreshToken, confluenceSourceConfig.getAuthenticationConfig().getOauth2Config().getRefreshToken()); + assertEquals(clientId, confluenceSourceConfig.getAuthenticationConfig().getOauth2Config().getClientId()); + assertEquals(clientSecret, confluenceSourceConfig.getAuthenticationConfig().getOauth2Config().getClientSecret()); + } +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceTest.java new file mode 100644 index 0000000000..ada00ab1d1 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceTest.java @@ -0,0 +1,118 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSetManager; +import org.opensearch.dataprepper.model.buffer.Buffer; +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.plugin.PluginFactory; +import org.opensearch.dataprepper.model.record.Record; +import org.opensearch.dataprepper.model.source.coordinator.enhanced.EnhancedSourceCoordinator; +import org.opensearch.dataprepper.plugins.source.confluence.configuration.AuthenticationConfig; +import org.opensearch.dataprepper.plugins.source.confluence.configuration.BasicConfig; +import org.opensearch.dataprepper.plugins.source.confluence.rest.auth.ConfluenceAuthConfig; +import org.opensearch.dataprepper.plugins.source.source_crawler.base.Crawler; +import org.opensearch.dataprepper.plugins.source.source_crawler.base.PluginExecutorServiceProvider; + +import java.util.concurrent.ExecutorService; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.atLeast; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.opensearch.dataprepper.plugins.source.confluence.rest.auth.ConfluenceOauthConfig.ACCESSIBLE_RESOURCES; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.BASIC; + +@ExtendWith(MockitoExtension.class) +public class ConfluenceSourceTest { + + @Mock + Buffer> buffer; + @Mock + AuthenticationConfig authenticationConfig; + @Mock + BasicConfig basicConfig; + @Mock + private PluginMetrics pluginMetrics; + @Mock + private ConfluenceSourceConfig confluenceSourceConfig; + @Mock + private ConfluenceAuthConfig jiraOauthConfig; + @Mock + private PluginFactory pluginFactory; + @Mock + private AcknowledgementSetManager acknowledgementSetManager; + @Mock + private Crawler crawler; + @Mock + private EnhancedSourceCoordinator sourceCooridinator; + @Mock + private PluginExecutorServiceProvider executorServiceProvider; + @Mock + private ExecutorService executorService; + + @Test + void initialization() { + when(executorServiceProvider.get()).thenReturn(executorService); + ConfluenceSource source = new ConfluenceSource(pluginMetrics, confluenceSourceConfig, jiraOauthConfig, pluginFactory, acknowledgementSetManager, crawler, executorServiceProvider); + assertNotNull(source); + } + + @Test + void testStart() { + when(executorServiceProvider.get()).thenReturn(executorService); + ConfluenceSource source = new ConfluenceSource(pluginMetrics, confluenceSourceConfig, jiraOauthConfig, pluginFactory, acknowledgementSetManager, crawler, executorServiceProvider); + when(confluenceSourceConfig.getAccountUrl()).thenReturn(ACCESSIBLE_RESOURCES); + when(confluenceSourceConfig.getAuthType()).thenReturn(BASIC); + when(confluenceSourceConfig.getAuthenticationConfig()).thenReturn(authenticationConfig); + when(authenticationConfig.getBasicConfig()).thenReturn(basicConfig); + when(basicConfig.getUsername()).thenReturn("Test Id"); + when(basicConfig.getPassword()).thenReturn("Test Credential"); + + source.setEnhancedSourceCoordinator(sourceCooridinator); + source.start(buffer); + verify(executorService, atLeast(1)).submit(any(Runnable.class)); + } + + @Test + void testStop() { + when(executorServiceProvider.get()).thenReturn(executorService); + ConfluenceSource source = new ConfluenceSource(pluginMetrics, confluenceSourceConfig, jiraOauthConfig, pluginFactory, acknowledgementSetManager, crawler, executorServiceProvider); + when(confluenceSourceConfig.getAccountUrl()).thenReturn(ACCESSIBLE_RESOURCES); + when(confluenceSourceConfig.getAuthType()).thenReturn(BASIC); + when(confluenceSourceConfig.getAuthenticationConfig()).thenReturn(authenticationConfig); + when(authenticationConfig.getBasicConfig()).thenReturn(basicConfig); + when(basicConfig.getUsername()).thenReturn("Test Id"); + when(basicConfig.getPassword()).thenReturn("Test Credential"); + + source.setEnhancedSourceCoordinator(sourceCooridinator); + source.start(buffer); + source.stop(); + verify(executorService).shutdownNow(); + } + + @Test + void testStop_WhenNotStarted() { + when(executorServiceProvider.get()).thenReturn(executorService); + ConfluenceSource source = new ConfluenceSource(pluginMetrics, confluenceSourceConfig, jiraOauthConfig, pluginFactory, acknowledgementSetManager, crawler, executorServiceProvider); + + source.stop(); + + verify(executorService, never()).shutdown(); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/exception/BadRequestExceptionTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/exception/BadRequestExceptionTest.java new file mode 100644 index 0000000000..4a38014021 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/exception/BadRequestExceptionTest.java @@ -0,0 +1,65 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.exception; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.opensearch.dataprepper.plugins.source.confluence.exception.BadRequestException; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.Mockito.mock; + +public class BadRequestExceptionTest { + private String message; + private Throwable throwable; + + @BeforeEach + void setUp() { + message = "Bad Request"; + throwable = mock(Throwable.class); + } + + @Nested + class MessageOnlyConstructor { + private BadRequestException createObjectUnderTest() { + return new BadRequestException(message); + } + + @Test + void getMessage_returns_message() { + assertEquals(createObjectUnderTest().getMessage(), message); + } + + @Test + void getCause_returns_null() { + assertNull(createObjectUnderTest().getCause()); + } + } + + @Nested + class MessageThrowableConstructor { + private BadRequestException createObjectUnderTest() { + return new BadRequestException(message, throwable); + } + + @Test + void getMessage_returns_message() { + assertEquals(createObjectUnderTest().getMessage(), message); + } + + @Test + void getCause_returns_throwable() { + assertEquals(createObjectUnderTest().getCause(), throwable); + } + } +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/exception/UnAuthorizedExceptionTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/exception/UnAuthorizedExceptionTest.java new file mode 100644 index 0000000000..a8a979d68f --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/exception/UnAuthorizedExceptionTest.java @@ -0,0 +1,65 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.exception; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.opensearch.dataprepper.plugins.source.confluence.exception.UnAuthorizedException; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.Mockito.mock; + +public class UnAuthorizedExceptionTest { + private String message; + private Throwable throwable; + + @BeforeEach + void setUp() { + message = "UnAuthorized Exception"; + throwable = mock(Throwable.class); + } + + @Nested + class MessageOnlyConstructor { + private UnAuthorizedException createObjectUnderTest() { + return new UnAuthorizedException(message); + } + + @Test + void getMessage_returns_message() { + assertEquals(createObjectUnderTest().getMessage(), message); + } + + @Test + void getCause_returns_null() { + assertNull(createObjectUnderTest().getCause()); + } + } + + @Nested + class MessageThrowableConstructor { + private UnAuthorizedException createObjectUnderTest() { + return new UnAuthorizedException(message, throwable); + } + + @Test + void getMessage_returns_message() { + assertEquals(createObjectUnderTest().getMessage(), message); + } + + @Test + void getCause_returns_throwable() { + assertEquals(createObjectUnderTest().getCause(), throwable); + } + } +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/IssueBeanTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/IssueBeanTest.java new file mode 100644 index 0000000000..3b3216a03a --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/IssueBeanTest.java @@ -0,0 +1,131 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.models; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.plugins.source.confluence.models.IssueBean; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.KEY; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.NAME; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.PROJECT; + +@ExtendWith(MockitoExtension.class) +public class IssueBeanTest { + + @Mock + private Map renderedFieldsTestObject; + + @Mock + private Map propertiesTestObject; + + @Mock + private Map namesTestObject; + + @Mock + private Map fieldsTestObject; + + private IssueBean issueBean; + + @BeforeEach + void setup() { + issueBean = new IssueBean(); + } + + @Test + public void testInitialization() { + assertNotNull(issueBean); + } + + @Test + public void testNull() { + assertNull(issueBean.getExpand()); + assertNull(issueBean.getId()); + assertNull(issueBean.getSelf()); + assertNull(issueBean.getKey()); + assertNull(issueBean.getRenderedFields()); + assertNull(issueBean.getProperties()); + assertNull(issueBean.getNames()); + assertNull(issueBean.getFields()); + } + + @Test + void testNullCases() { + assertNull(issueBean.getProject()); + assertNull(issueBean.getProjectName()); + assertEquals(issueBean.getUpdatedTimeMillis(), 0); + } + + @Test + void testGivenDateField() { + Map fieldsTestObject = new HashMap<>(); + fieldsTestObject.put("created", "2024-07-06T21:12:23.437-0700"); + fieldsTestObject.put("updated", "2022-07-06T21:12:23.106-0700"); + issueBean.setFields(fieldsTestObject); + assertEquals(issueBean.getCreatedTimeMillis(), 1720325543000L); + assertEquals(issueBean.getUpdatedTimeMillis(), 1657167143000L); + } + + @Test + public void testStringSettersAndGetters() { + String self = "selfTest"; + String key = "keyTest"; + String id = "idTest"; + String expand = "expandTest"; + + issueBean.setExpand(expand); + assertEquals(issueBean.getExpand(), expand); + issueBean.setId(id); + assertEquals(issueBean.getId(), id); + issueBean.setSelf(self); + assertEquals(issueBean.getSelf(), self); + issueBean.setKey(key); + assertEquals(issueBean.getKey(), key); + } + + @Test + public void testMapSettersAndGetters() { + + issueBean.setRenderedFields(renderedFieldsTestObject); + assertEquals(issueBean.getRenderedFields(), renderedFieldsTestObject); + issueBean.setProperties(propertiesTestObject); + assertEquals(issueBean.getProperties(), propertiesTestObject); + issueBean.setNames(namesTestObject); + assertEquals(issueBean.getNames(), namesTestObject); + issueBean.setFields(fieldsTestObject); + assertEquals(issueBean.getFields(), fieldsTestObject); + } + + @Test + public void testFieldPropertyGetters() { + Map fieldsTestObject = new HashMap<>(); + Map projectTestObject = new HashMap<>(); + String projectName = "name of project"; + String projectKey = "PROJKEY"; + projectTestObject.put(KEY, projectKey); + projectTestObject.put(NAME, projectName); + fieldsTestObject.put(PROJECT, projectTestObject); + + issueBean.setFields(fieldsTestObject); + assertEquals(projectKey, issueBean.getProject()); + assertEquals(projectName, issueBean.getProjectName()); + } + +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/SearchResultsTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/SearchResultsTest.java new file mode 100644 index 0000000000..44321d2634 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/SearchResultsTest.java @@ -0,0 +1,98 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.models; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.plugins.source.confluence.models.IssueBean; +import org.opensearch.dataprepper.plugins.source.confluence.models.SearchResults; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +@ExtendWith(MockitoExtension.class) +public class SearchResultsTest { + + private SearchResults searchResults; + private final ObjectMapper objectMapper = new ObjectMapper(); + + @BeforeEach + public void setUp() throws JsonProcessingException { + String state = "{}"; + searchResults = objectMapper.readValue(state, SearchResults.class); + } + + @Test + public void testConstructor() { + assertNotNull(searchResults); + + assertNull(searchResults.getExpand()); + assertNull(searchResults.getStartAt()); + assertNull(searchResults.getMaxResults()); + assertNull(searchResults.getTotal()); + assertNull(searchResults.getIssues()); + } + + @Test + public void testGetters() throws JsonProcessingException { + String expand = "expandTest"; + Integer startAt = 1; + Integer maxResults = 100; + Integer total = 10; + List testIssues = new ArrayList<>(); + IssueBean issue1 = new IssueBean(); + IssueBean issue2 = new IssueBean(); + issue1.setId("issue 1"); + issue2.setId("issue 2"); + testIssues.add(issue1); + testIssues.add(issue2); + + + Map searchResultsMap = new HashMap<>(); + searchResultsMap.put("expand", expand); + searchResultsMap.put("startAt", startAt); + searchResultsMap.put("maxResults", maxResults); + searchResultsMap.put("total", total); + searchResultsMap.put("issues", testIssues); + + String jsonString = objectMapper.writeValueAsString(searchResultsMap); + + searchResults = objectMapper.readValue(jsonString, SearchResults.class); + + assertEquals(searchResults.getExpand(), expand); + assertEquals(searchResults.getStartAt(), startAt); + assertEquals(searchResults.getMaxResults(), maxResults); + assertEquals(searchResults.getTotal(), total); + + List returnedIssues = searchResults.getIssues(); + assertNotNull(returnedIssues); + assertEquals(testIssues.size(), returnedIssues.size()); + + for (int i = 0; i < testIssues.size(); i++) { + IssueBean originalIssue = testIssues.get(i); + IssueBean returnedIssue = returnedIssues.get(i); + + assertEquals(originalIssue.getId(), returnedIssue.getId()); + } + } + + +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/BasicAuthInterceptorTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/BasicAuthInterceptorTest.java new file mode 100644 index 0000000000..68d05b47aa --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/BasicAuthInterceptorTest.java @@ -0,0 +1,80 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.rest; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceSourceConfig; +import org.opensearch.dataprepper.plugins.source.confluence.configuration.AuthenticationConfig; +import org.opensearch.dataprepper.plugins.source.confluence.configuration.BasicConfig; +import org.opensearch.dataprepper.plugins.source.confluence.rest.BasicAuthInterceptor; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpRequest; +import org.springframework.http.client.ClientHttpRequestExecution; +import org.springframework.http.client.ClientHttpResponse; + +import java.io.IOException; +import java.util.Base64; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +public class BasicAuthInterceptorTest { + + @Mock + AuthenticationConfig authenticationConfig; + @Mock + BasicConfig basicConfig; + @Mock + private HttpRequest mockRequest; + @Mock + private ClientHttpRequestExecution mockExecution; + @Mock + private ClientHttpResponse mockResponse; + @Mock + private ConfluenceSourceConfig mockConfig; + @Mock + private HttpHeaders mockHeaders; + + private BasicAuthInterceptor interceptor; + + @BeforeEach + void setUp() { + when(mockConfig.getAuthenticationConfig()).thenReturn(authenticationConfig); + when(authenticationConfig.getBasicConfig()).thenReturn(basicConfig); + when(basicConfig.getUsername()).thenReturn("testUser"); + when(basicConfig.getPassword()).thenReturn("testPassword"); + when(mockRequest.getHeaders()).thenReturn(mockHeaders); + interceptor = new BasicAuthInterceptor(mockConfig); + } + + @Test + void testInterceptAddsAuthorizationHeader() throws IOException { + when(mockExecution.execute(any(HttpRequest.class), any(byte[].class))).thenReturn(mockResponse); + + ClientHttpResponse response = interceptor.intercept(mockRequest, new byte[0], mockExecution); + + verify(mockHeaders).set(eq(HttpHeaders.AUTHORIZATION), argThat(value -> + value.startsWith("Basic ") && + new String(Base64.getDecoder().decode(value.substring(6))).equals("testUser:testPassword") + )); + assertEquals(mockResponse, response); + } + +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClientTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClientTest.java new file mode 100644 index 0000000000..58863e85e5 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClientTest.java @@ -0,0 +1,150 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.rest; + +import com.fasterxml.jackson.core.JsonProcessingException; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceServiceTest; +import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceSourceConfig; +import org.opensearch.dataprepper.plugins.source.confluence.exception.BadRequestException; +import org.opensearch.dataprepper.plugins.source.confluence.exception.UnAuthorizedException; +import org.opensearch.dataprepper.plugins.source.confluence.models.SearchResults; +import org.opensearch.dataprepper.plugins.source.confluence.rest.auth.ConfluenceAuthConfig; +import org.opensearch.dataprepper.plugins.source.confluence.rest.auth.ConfluenceAuthFactory; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.RestTemplate; + +import java.net.URI; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.BASIC; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.OAUTH2; + +@ExtendWith(MockitoExtension.class) +public class ConfluenceRestClientTest { + + @Mock + private StringBuilder jql; + + @Mock + private RestTemplate restTemplate; + + @Mock + private ConfluenceAuthConfig authConfig; + + private static Stream provideHttpStatusCodesWithExceptionClass() { + return Stream.of( + Arguments.of(HttpStatus.FORBIDDEN, UnAuthorizedException.class), + Arguments.of(HttpStatus.UNAUTHORIZED, RuntimeException.class), + Arguments.of(HttpStatus.TOO_MANY_REQUESTS, RuntimeException.class), + Arguments.of(HttpStatus.INSUFFICIENT_STORAGE, RuntimeException.class) + ); + } + + @ParameterizedTest + @ValueSource(strings = {"basic-auth-jira-pipeline.yaml"}) + public void testFetchingJiraIssue(String configFileName) { + String exampleTicketResponse = "{\"id\":\"123\",\"key\":\"key\",\"self\":\"https://example.com/rest/api/2/issue/123\"}"; + doReturn(new ResponseEntity<>(exampleTicketResponse, HttpStatus.OK)).when(restTemplate).getForEntity(any(URI.class), any(Class.class)); + ConfluenceSourceConfig confluenceSourceConfig = ConfluenceServiceTest.createJiraConfigurationFromYaml(configFileName); + ConfluenceAuthConfig authConfig = new ConfluenceAuthFactory(confluenceSourceConfig).getObject(); + ConfluenceRestClient confluenceRestClient = new ConfluenceRestClient(restTemplate, authConfig); + String ticketDetails = confluenceRestClient.getIssue("key"); + assertEquals(exampleTicketResponse, ticketDetails); + } + + @ParameterizedTest + @MethodSource("provideHttpStatusCodesWithExceptionClass") + void testInvokeRestApiTokenExpired(HttpStatus statusCode, Class expectedExceptionType) { + ConfluenceRestClient confluenceRestClient = new ConfluenceRestClient(restTemplate, authConfig); + confluenceRestClient.setSleepTimeMultiplier(1); + when(authConfig.getUrl()).thenReturn("https://example.com/rest/api/2/issue/key"); + when(restTemplate.getForEntity(any(URI.class), any(Class.class))).thenThrow(new HttpClientErrorException(statusCode)); + assertThrows(expectedExceptionType, () -> confluenceRestClient.getIssue("key")); + } + + @Test + void testInvokeRestApiTokenExpiredInterruptException() throws InterruptedException { + ConfluenceRestClient confluenceRestClient = new ConfluenceRestClient(restTemplate, authConfig); + when(authConfig.getUrl()).thenReturn("https://example.com/rest/api/2/issue/key"); + when(restTemplate.getForEntity(any(URI.class), any(Class.class))).thenThrow(new HttpClientErrorException(HttpStatus.TOO_MANY_REQUESTS)); + confluenceRestClient.setSleepTimeMultiplier(100000); + + Thread testThread = new Thread(() -> { + assertThrows(InterruptedException.class, () -> { + try { + confluenceRestClient.getIssue("key"); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + }); + testThread.start(); + Thread.sleep(100); + testThread.interrupt(); + } + + @Test + public void testGetAllIssuesOauth2() throws JsonProcessingException { + List issueType = new ArrayList<>(); + List issueStatus = new ArrayList<>(); + List projectKey = new ArrayList<>(); + issueType.add("Task"); + ConfluenceSourceConfig confluenceSourceConfig = ConfluenceServiceTest.createJiraConfiguration(OAUTH2, issueType, issueStatus, projectKey); + ConfluenceRestClient confluenceRestClient = new ConfluenceRestClient(restTemplate, authConfig); + SearchResults mockSearchResults = mock(SearchResults.class); + doReturn("http://mock-service.jira.com/").when(authConfig).getUrl(); + doReturn(new ResponseEntity<>(mockSearchResults, HttpStatus.OK)).when(restTemplate).getForEntity(any(URI.class), any(Class.class)); + SearchResults results = confluenceRestClient.getAllIssues(jql, 0, confluenceSourceConfig); + assertNotNull(results); + } + + @Test + public void testGetAllIssuesBasic() throws JsonProcessingException { + List issueType = new ArrayList<>(); + List issueStatus = new ArrayList<>(); + List projectKey = new ArrayList<>(); + issueType.add("Task"); + ConfluenceSourceConfig confluenceSourceConfig = ConfluenceServiceTest.createJiraConfiguration(BASIC, issueType, issueStatus, projectKey); + ConfluenceRestClient confluenceRestClient = new ConfluenceRestClient(restTemplate, authConfig); + SearchResults mockSearchResults = mock(SearchResults.class); + when(authConfig.getUrl()).thenReturn("https://example.com/"); + doReturn(new ResponseEntity<>(mockSearchResults, HttpStatus.OK)).when(restTemplate).getForEntity(any(URI.class), any(Class.class)); + SearchResults results = confluenceRestClient.getAllIssues(jql, 0, confluenceSourceConfig); + assertNotNull(results); + } + + @Test + public void testRestApiAddressValidation() throws JsonProcessingException { + when(authConfig.getUrl()).thenReturn("https://224.0.0.1/"); + ConfluenceRestClient confluenceRestClient = new ConfluenceRestClient(restTemplate, authConfig); + assertThrows(BadRequestException.class, () -> confluenceRestClient.getIssue("TEST-1")); + } + +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/CustomRestTemplateConfigTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/CustomRestTemplateConfigTest.java new file mode 100644 index 0000000000..23297eb380 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/CustomRestTemplateConfigTest.java @@ -0,0 +1,105 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.rest; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.model.plugin.PluginConfigVariable; +import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceSourceConfig; +import org.opensearch.dataprepper.plugins.source.confluence.configuration.AuthenticationConfig; +import org.opensearch.dataprepper.plugins.source.confluence.configuration.BasicConfig; +import org.opensearch.dataprepper.plugins.source.confluence.configuration.Oauth2Config; +import org.opensearch.dataprepper.plugins.source.confluence.rest.auth.ConfluenceAuthConfig; +import org.springframework.http.client.ClientHttpRequestInterceptor; +import org.springframework.http.client.InterceptingClientHttpRequestFactory; +import org.springframework.web.client.RestTemplate; + +import java.util.List; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.when; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.BASIC; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.OAUTH2; + +@ExtendWith(MockitoExtension.class) +class CustomRestTemplateConfigTest { + + private CustomRestTemplateConfig config; + + @Mock + private ConfluenceSourceConfig mockSourceConfig; + + @Mock + private ConfluenceAuthConfig mockAuthConfig; + + @Mock + private BasicConfig mockBasicConfig; + + @Mock + private Oauth2Config mockOauth2Config; + + @Mock + private PluginConfigVariable accessTokenPluginConfigVariable; + + @Mock + private PluginConfigVariable refreshTokenPluginConfigVariable; + + @Mock + private AuthenticationConfig mockAuthenticationConfig; + + private static Stream provideAuthTypeAndExpectedInterceptorType() { + return Stream.of( + Arguments.of(OAUTH2, OAuth2RequestInterceptor.class), + Arguments.of(BASIC, BasicAuthInterceptor.class), + Arguments.of("Default", BasicAuthInterceptor.class), + Arguments.of(null, BasicAuthInterceptor.class) + ); + } + + @BeforeEach + void setUp() { + config = new CustomRestTemplateConfig(); + } + + @ParameterizedTest + @MethodSource("provideAuthTypeAndExpectedInterceptorType") + void testBasicAuthRestTemplateWithOAuth2(String authType, Class interceptorClassType) { + when(mockSourceConfig.getAuthType()).thenReturn(authType); + lenient().when(mockSourceConfig.getAuthenticationConfig()).thenReturn(mockAuthenticationConfig); + lenient().when(mockAuthenticationConfig.getOauth2Config()).thenReturn(mockOauth2Config); + lenient().when(mockOauth2Config.getAccessToken()).thenReturn(accessTokenPluginConfigVariable); + lenient().when(mockOauth2Config.getRefreshToken()).thenReturn(refreshTokenPluginConfigVariable); + lenient().when(accessTokenPluginConfigVariable.getValue()).thenReturn("accessToken"); + lenient().when(mockOauth2Config.getClientId()).thenReturn("clientId"); + lenient().when(mockOauth2Config.getClientSecret()).thenReturn("clientSecret"); + lenient().when(mockAuthenticationConfig.getBasicConfig()).thenReturn(mockBasicConfig); + lenient().when(mockBasicConfig.getUsername()).thenReturn("username"); + lenient().when(mockBasicConfig.getPassword()).thenReturn("password"); + + RestTemplate restTemplate = config.basicAuthRestTemplate(mockSourceConfig, mockAuthConfig); + assertNotNull(restTemplate); + assertInstanceOf(InterceptingClientHttpRequestFactory.class, restTemplate.getRequestFactory()); + List interceptors = restTemplate.getInterceptors(); + assertEquals(1, interceptors.size()); + assertInstanceOf(interceptorClassType, interceptors.get(0)); + } + +} + diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/OAuth2RequestInterceptorTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/OAuth2RequestInterceptorTest.java new file mode 100644 index 0000000000..554b97e674 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/OAuth2RequestInterceptorTest.java @@ -0,0 +1,66 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.rest; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.plugins.source.confluence.rest.auth.ConfluenceOauthConfig; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpRequest; +import org.springframework.http.client.ClientHttpRequestExecution; +import org.springframework.http.client.ClientHttpResponse; + +import java.io.IOException; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +public class OAuth2RequestInterceptorTest { + + @Mock + private HttpRequest mockRequest; + + @Mock + private ClientHttpRequestExecution mockExecution; + + @Mock + private ClientHttpResponse mockResponse; + + @Mock + private ConfluenceOauthConfig mockConfig; + + @Mock + private HttpHeaders mockHeaders; + + private OAuth2RequestInterceptor interceptor; + + @BeforeEach + void setUp() { + when(mockConfig.getAccessToken()).thenReturn("testAccessToken"); + when(mockRequest.getHeaders()).thenReturn(mockHeaders); + interceptor = new OAuth2RequestInterceptor(mockConfig); + } + + + @Test + void testInterceptAddsAuthorizationHeader() throws IOException { + when(mockExecution.execute(any(HttpRequest.class), any(byte[].class))).thenReturn(mockResponse); + ClientHttpResponse response = interceptor.intercept(mockRequest, new byte[0], mockExecution); + verify(mockHeaders).setBearerAuth("testAccessToken"); + assertEquals(mockResponse, response); + } +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceAuthFactoryTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceAuthFactoryTest.java new file mode 100644 index 0000000000..60fdbdcfd1 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceAuthFactoryTest.java @@ -0,0 +1,74 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.rest.auth; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.model.plugin.PluginConfigVariable; +import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceSourceConfig; +import org.opensearch.dataprepper.plugins.source.confluence.configuration.AuthenticationConfig; +import org.opensearch.dataprepper.plugins.source.confluence.configuration.Oauth2Config; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.mockito.Mockito.when; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.OAUTH2; + +@ExtendWith(MockitoExtension.class) +public class ConfluenceAuthFactoryTest { + + @Mock + private ConfluenceSourceConfig sourceConfig; + + @Mock + private AuthenticationConfig authenticationConfig; + + @Mock + private Oauth2Config oauth2Config; + + @Mock + private PluginConfigVariable accessTokenPluginConfigVariable; + + @Mock + private PluginConfigVariable refreshTokenPluginConfigVariable; + + private ConfluenceAuthFactory confluenceAuthFactory; + + @BeforeEach + void setUp() { + confluenceAuthFactory = new ConfluenceAuthFactory(sourceConfig); + } + + @Test + void testGetObjectOauth2() { + when(sourceConfig.getAuthType()).thenReturn(OAUTH2); + when(sourceConfig.getAuthenticationConfig()).thenReturn(authenticationConfig); + when(authenticationConfig.getOauth2Config()).thenReturn(oauth2Config); + when(oauth2Config.getRefreshToken()).thenReturn(refreshTokenPluginConfigVariable); + when(oauth2Config.getAccessToken()).thenReturn(accessTokenPluginConfigVariable); + when(accessTokenPluginConfigVariable.getValue()).thenReturn("mockRefreshToken"); + assertInstanceOf(ConfluenceOauthConfig.class, confluenceAuthFactory.getObject()); + } + + @Test + void testGetObjectBasicAuth() { + when(sourceConfig.getAccountUrl()).thenReturn("https://example.com"); + assertInstanceOf(ConfluenceBasicAuthConfig.class, confluenceAuthFactory.getObject()); + } + + @Test + void testGetObjectType() { + assertEquals(ConfluenceAuthConfig.class, confluenceAuthFactory.getObjectType()); + } +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/JiraBasicAuthConfigTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/JiraBasicAuthConfigTest.java new file mode 100644 index 0000000000..ae6f8f3325 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/JiraBasicAuthConfigTest.java @@ -0,0 +1,48 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.rest.auth; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceSourceConfig; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +public class JiraBasicAuthConfigTest { + + String url = "https://example.com"; + @Mock + private ConfluenceSourceConfig confluenceSourceConfig; + private ConfluenceBasicAuthConfig jiraBasicAuthConfig; + + @BeforeEach + void setUp() { + when(confluenceSourceConfig.getAccountUrl()).thenReturn(url); + jiraBasicAuthConfig = new ConfluenceBasicAuthConfig(confluenceSourceConfig); + } + + @Test + void testGetUrl() { + assertEquals(jiraBasicAuthConfig.getUrl(), url + '/'); + + } + + @Test + void DoNothingForBasicAuthentication() { + jiraBasicAuthConfig.initCredentials(); + jiraBasicAuthConfig.renewCredentials(); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/JiraOauthConfigTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/JiraOauthConfigTest.java new file mode 100644 index 0000000000..7d7ef8aae4 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/JiraOauthConfigTest.java @@ -0,0 +1,187 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.rest.auth; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.model.plugin.PluginConfigVariable; +import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceServiceTest; +import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceSourceConfig; +import org.opensearch.dataprepper.plugins.source.confluence.configuration.Oauth2Config; +import org.opensearch.dataprepper.plugins.source.confluence.exception.UnAuthorizedException; +import org.opensearch.dataprepper.test.helper.ReflectivelySetField; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.RestTemplate; + +import java.time.Instant; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.RETRY_ATTEMPT; + +@ExtendWith(MockitoExtension.class) +public class JiraOauthConfigTest { + + @Mock + RestTemplate restTemplateMock; + + + ConfluenceSourceConfig confluenceSourceConfig; + + @Mock + PluginConfigVariable accessTokenVariable; + + @BeforeEach + void setUp() { + confluenceSourceConfig = ConfluenceServiceTest.createJiraConfigurationFromYaml("oauth2-auth-jira-pipeline.yaml"); + } + + @Test + void testRenewToken() throws InterruptedException { + Instant testStartTime = Instant.now(); + Map firstMockResponseMap = Map.of("access_token", "first_mock_access_token", + "refresh_token", "first_mock_refresh_token", + "expires_in", 3600); + ConfluenceOauthConfig jiraOauthConfig = new ConfluenceOauthConfig(confluenceSourceConfig); + when(restTemplateMock.postForEntity(any(String.class), any(HttpEntity.class), any(Class.class))) + .thenReturn(new ResponseEntity<>(firstMockResponseMap, HttpStatus.OK)); + jiraOauthConfig.restTemplate = restTemplateMock; + ExecutorService executor = Executors.newFixedThreadPool(2); + Future firstCall = executor.submit(jiraOauthConfig::renewCredentials); + Future secondCall = executor.submit(jiraOauthConfig::renewCredentials); + while (!firstCall.isDone() || !secondCall.isDone()) { + // Do nothing. Wait for the calls to complete + Thread.sleep(10); + } + executor.shutdown(); + assertNotNull(jiraOauthConfig.getAccessToken()); + assertNotNull(jiraOauthConfig.getExpireTime()); + assertEquals(jiraOauthConfig.getRefreshToken(), "first_mock_refresh_token"); + assertEquals(jiraOauthConfig.getExpiresInSeconds(), 3600); + assertEquals(jiraOauthConfig.getAccessToken(), "first_mock_access_token"); + assertTrue(jiraOauthConfig.getExpireTime().isAfter(testStartTime)); + Instant expectedNewExpireTime = Instant.ofEpochMilli(testStartTime.toEpochMilli() + 3601 * 1000); + assertTrue(jiraOauthConfig.getExpireTime().isBefore(expectedNewExpireTime), + String.format("Expected that %s time should be before %s", jiraOauthConfig.getExpireTime(), expectedNewExpireTime)); + verify(restTemplateMock, times(1)).postForEntity(any(String.class), any(HttpEntity.class), any(Class.class)); + + } + + @Test + void testFailedToRenewAccessToken() throws NoSuchFieldException, IllegalAccessException { + ConfluenceOauthConfig jiraOauthConfig = new ConfluenceOauthConfig(confluenceSourceConfig); + Oauth2Config oauth2Config = confluenceSourceConfig.getAuthenticationConfig().getOauth2Config(); + ReflectivelySetField.setField(Oauth2Config.class, oauth2Config, "accessToken", accessTokenVariable); + when(restTemplateMock.postForEntity(any(String.class), any(HttpEntity.class), any(Class.class))) + .thenThrow(HttpClientErrorException.class); + jiraOauthConfig.restTemplate = restTemplateMock; + assertThrows(RuntimeException.class, jiraOauthConfig::renewCredentials); + verify(oauth2Config.getAccessToken(), times(0)) + .refresh(); + } + + @Test + void testFailedToRenewAccessToken_with_unauthorized_and_trigger_secrets_refresh() + throws NoSuchFieldException, IllegalAccessException { + ConfluenceOauthConfig jiraOauthConfig = new ConfluenceOauthConfig(confluenceSourceConfig); + Oauth2Config oauth2Config = confluenceSourceConfig.getAuthenticationConfig().getOauth2Config(); + ReflectivelySetField.setField(Oauth2Config.class, oauth2Config, "accessToken", accessTokenVariable); + HttpClientErrorException unAuthorizedException = new HttpClientErrorException(HttpStatus.UNAUTHORIZED); + when(restTemplateMock.postForEntity(any(String.class), any(HttpEntity.class), any(Class.class))) + .thenThrow(unAuthorizedException); + jiraOauthConfig.restTemplate = restTemplateMock; + assertThrows(RuntimeException.class, jiraOauthConfig::renewCredentials); + verify(oauth2Config.getAccessToken(), times(1)) + .refresh(); + } + + + @Test + void testGetJiraAccountCloudId() throws InterruptedException { + Map mockGetCallResponse = new HashMap<>(); + mockGetCallResponse.put("id", "test_cloud_id"); + when(restTemplateMock.exchange(any(String.class), any(HttpMethod.class), any(HttpEntity.class), any(Class.class))) + .thenReturn(new ResponseEntity<>(List.of(mockGetCallResponse), HttpStatus.OK)); + ConfluenceOauthConfig jiraOauthConfig = new ConfluenceOauthConfig(confluenceSourceConfig); + jiraOauthConfig.restTemplate = restTemplateMock; + + ExecutorService executor = Executors.newFixedThreadPool(2); + Future firstCall = executor.submit(jiraOauthConfig::getUrl); + Future secondCall = executor.submit(jiraOauthConfig::getUrl); + while (!firstCall.isDone() || !secondCall.isDone()) { + // Do nothing. Wait for the calls to complete + Thread.sleep(10); + } + executor.shutdown(); + + assertEquals("test_cloud_id", jiraOauthConfig.getJiraAccountCloudId()); + assertEquals("https://api.atlassian.com/ex/jira/test_cloud_id/", jiraOauthConfig.getUrl()); + //calling second time shouldn't trigger rest call + jiraOauthConfig.getUrl(); + verify(restTemplateMock, times(1)) + .exchange(any(String.class), any(HttpMethod.class), any(HttpEntity.class), any(Class.class)); + } + + @Test + void testGetJiraAccountCloudIdUnauthorizedCase() { + + when(restTemplateMock.exchange(any(String.class), any(HttpMethod.class), any(HttpEntity.class), any(Class.class))) + .thenThrow(new HttpClientErrorException(HttpStatus.UNAUTHORIZED)); + Map mockRenewTokenResponse = Map.of("access_token", "first_mock_access_token", + "refresh_token", "first_mock_refresh_token", + "expires_in", 3600); + when(restTemplateMock.postForEntity(any(String.class), any(HttpEntity.class), any(Class.class))) + .thenReturn(new ResponseEntity<>(mockRenewTokenResponse, HttpStatus.OK)); + ConfluenceOauthConfig jiraOauthConfig = new ConfluenceOauthConfig(confluenceSourceConfig); + jiraOauthConfig.restTemplate = restTemplateMock; + + + assertThrows(UnAuthorizedException.class, () -> jiraOauthConfig.initCredentials()); + verify(restTemplateMock, times(6)) + .exchange(any(String.class), any(HttpMethod.class), any(HttpEntity.class), any(Class.class)); + verify(restTemplateMock, times(1)) + .postForEntity(any(String.class), any(HttpEntity.class), any(Class.class)); + + } + + @Test + void testFailedToGetCloudId() { + when(restTemplateMock.exchange(any(String.class), any(HttpMethod.class), any(HttpEntity.class), any(Class.class))) + .thenThrow(new HttpClientErrorException(HttpStatus.UNAUTHORIZED)) + .thenThrow(HttpClientErrorException.class); + ConfluenceOauthConfig jiraOauthConfig = new ConfluenceOauthConfig(confluenceSourceConfig); + jiraOauthConfig.restTemplate = restTemplateMock; + assertThrows(RuntimeException.class, jiraOauthConfig::getUrl); + for (int i = 0; i <= RETRY_ATTEMPT; i++) { + assertThrows(RuntimeException.class, jiraOauthConfig::getUrl); + } + } + +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/utils/AddressValidationTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/utils/AddressValidationTest.java new file mode 100644 index 0000000000..0077129427 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/utils/AddressValidationTest.java @@ -0,0 +1,105 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.utils; + +import org.junit.jupiter.api.Test; +import org.opensearch.dataprepper.plugins.source.confluence.exception.BadRequestException; +import org.opensearch.dataprepper.plugins.source.confluence.utils.AddressValidation; +import org.opensearch.dataprepper.plugins.source.confluence.utils.Constants; +import org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants; + +import java.net.InetAddress; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.UnknownHostException; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class AddressValidationTest { + + @Test + void testInstanceCreation() { + assertNotNull(new AddressValidation()); + assertNotNull(new Constants()); + assertNotNull(new JqlConstants()); + } + + @Test + void testGetInetAddress() { + String testUrl = "https://www.amazon.com"; + System.out.print("Test output"); + System.out.print(AddressValidation.getInetAddress(testUrl)); + } + + @Test + void testGetInetAddressWithMalformedUrl() { + String testUrl = "XXXXXXXXXXXXXXXXXXXXXX"; + assertThrows(BadRequestException.class, () -> AddressValidation.getInetAddress(testUrl)); + } + + @Test + void testGetInetAddressWithUnknownHost() { + String testUrl = "https://www.thisurldoesntexist1384276t5917278481073.com"; + assertThrows(BadRequestException.class, () -> AddressValidation.getInetAddress(testUrl)); + } + + @Test + void testGetInetAddressWithNullUrl() { + String testUrl = null; + assertThrows(BadRequestException.class, () -> AddressValidation.getInetAddress(testUrl)); + } + + + @Test + void testValidateInetAddressAnyLocalAddress() throws UnknownHostException { + InetAddress wildcardAddress = InetAddress.getByName("0.0.0.0"); + assertThrows(BadRequestException.class, () -> AddressValidation.validateInetAddress(wildcardAddress)); + } + + @Test + void testValidateInetAddressMulticastAddress() throws UnknownHostException { + InetAddress multicastAddress = InetAddress.getByName("224.0.0.1"); + assertThrows(BadRequestException.class, () -> AddressValidation.validateInetAddress(multicastAddress)); + } + + @Test + void testValidateInetAddressLinkLocalAddress() throws UnknownHostException { + InetAddress linkLocalAddress = InetAddress.getByName("169.254.1.1"); + assertThrows(BadRequestException.class, () -> AddressValidation.validateInetAddress(linkLocalAddress)); + } + + @Test + void testValidateInetAddressSiteLocalAddress() throws UnknownHostException { + InetAddress siteLocalAddress = InetAddress.getByName("10.0.0.1"); + assertThrows(BadRequestException.class, () -> AddressValidation.validateInetAddress(siteLocalAddress)); + } + + @Test + void testValidateInetAddressLoopbackAddress() throws UnknownHostException { + InetAddress loopbackAddress = InetAddress.getByName("127.0.0.1"); + assertThrows(BadRequestException.class, () -> AddressValidation.validateInetAddress(loopbackAddress)); + } + + @Test + void testValidateInetAddressValidAddress() throws UnknownHostException, MalformedURLException { + InetAddress validAddress = InetAddress.getByName(new URL("https://www.amazon.com").getHost()); + assertDoesNotThrow(() -> AddressValidation.validateInetAddress(validAddress)); + } + + @Test + void testValidateInetAddressNullAddress() { + InetAddress nullAddress = null; + assertThrows(NullPointerException.class, () -> AddressValidation.validateInetAddress(nullAddress)); + } + +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/utils/ConfluenceContentTypeTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/utils/ConfluenceContentTypeTest.java new file mode 100644 index 0000000000..a17c617719 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/utils/ConfluenceContentTypeTest.java @@ -0,0 +1,36 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.utils; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +public class ConfluenceContentTypeTest { + @Test + void testEnumConstants() { + assertNotNull(ConfluenceContentType.PROJECT); + assertNotNull(ConfluenceContentType.ISSUE); + assertNotNull(ConfluenceContentType.COMMENT); + assertNotNull(ConfluenceContentType.ATTACHMENT); + assertNotNull(ConfluenceContentType.WORKLOG); + } + + @Test + void testTypeValues() { + assertEquals("PROJECT", ConfluenceContentType.PROJECT.getType()); + assertEquals("ISSUE", ConfluenceContentType.ISSUE.getType()); + assertEquals("COMMENT", ConfluenceContentType.COMMENT.getType()); + assertEquals("ATTACHMENT", ConfluenceContentType.ATTACHMENT.getType()); + assertEquals("WORKLOG", ConfluenceContentType.WORKLOG.getType()); + } +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/utils/MockPluginConfigVariableImpl.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/utils/MockPluginConfigVariableImpl.java new file mode 100644 index 0000000000..d474ac5002 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/utils/MockPluginConfigVariableImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.utils; + +import org.opensearch.dataprepper.model.plugin.PluginConfigVariable; + +/** + * Mock implementation of PluginConfigVariable interface used only for Unit Testing. + */ +public class MockPluginConfigVariableImpl implements PluginConfigVariable { + + private Object defaultValue; + + public MockPluginConfigVariableImpl(Object defaultValue) { + this.defaultValue = defaultValue; + } + + @Override + public Object getValue() { + return null; + } + + @Override + public void setValue(Object someValue) { + this.defaultValue = someValue; + } + + @Override + public void refresh() { + } + + @Override + public boolean isUpdatable() { + return true; + } +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/resources/basic-auth-jira-pipeline.yaml b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/resources/basic-auth-jira-pipeline.yaml new file mode 100644 index 0000000000..0bfa6384e8 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/resources/basic-auth-jira-pipeline.yaml @@ -0,0 +1,6 @@ +hosts: ["https://jira.com/"] +authentication: + basic: + username: "jiraId" + password: "jiraApiKey" + diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/resources/oauth2-auth-jira-pipeline.yaml b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/resources/oauth2-auth-jira-pipeline.yaml new file mode 100644 index 0000000000..7a4afd3abf --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/resources/oauth2-auth-jira-pipeline.yaml @@ -0,0 +1,6 @@ +hosts: [ "https://jira.com/" ] +authentication: + oauth2: + client_id: "client_id" + client_secret: "client_secret" + diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java index 92420ac319..6d16e0f4a6 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java @@ -53,6 +53,7 @@ public class JiraRestClient { private static final String SEARCH_CALL_LATENCY_TIMER = "searchCallLatency"; private static final String PROJECTS_FETCH_LATENCY_TIMER = "projectFetchLatency"; private static final String ISSUES_REQUESTED = "issuesRequested"; + private int sleepTimeMultiplier = 1000; private final RestTemplate restTemplate; private final JiraAuthConfig authConfig; private final Timer ticketFetchLatencyTimer; @@ -60,7 +61,6 @@ public class JiraRestClient { private final Timer projectFetchLatencyTimer; private final Counter issuesRequestedCounter; private final PluginMetrics jiraPluginMetrics = PluginMetrics.fromNames("jiraRestClient", "aws"); - private int sleepTimeMultiplier = 1000; public JiraRestClient(RestTemplate restTemplate, JiraAuthConfig authConfig) { this.restTemplate = restTemplate; @@ -126,7 +126,8 @@ private ResponseEntity invokeRestApi(URI uri, Class responseType) thro log.error(NOISY, "Token expired. We will try to renew the tokens now", ex); authConfig.renewCredentials(); } else if (statusCode == HttpStatus.TOO_MANY_REQUESTS) { - log.error(NOISY, "Hitting API rate limit. Backing off with sleep timer.", ex); + log.error(NOISY, "Hitting API rate limit. Backing off with sleep timer for {} seconds", + RETRY_ATTEMPT_SLEEP_TIME.get(retryCount)); } try { Thread.sleep((long) RETRY_ATTEMPT_SLEEP_TIME.get(retryCount) * sleepTimeMultiplier); diff --git a/settings.gradle b/settings.gradle index d86bc7e1da..bee7e45f28 100644 --- a/settings.gradle +++ b/settings.gradle @@ -189,4 +189,5 @@ include 'data-prepper-plugins:opensearch-api-source' include 'data-prepper-plugins:saas-source-plugins' include 'data-prepper-plugins:saas-source-plugins:source-crawler' include 'data-prepper-plugins:saas-source-plugins:jira-source' +include 'data-prepper-plugins:saas-source-plugins:confluence-source' From 98660a17ce75b14fe3c327efcb2105bb84b11d28 Mon Sep 17 00:00:00 2001 From: Santhosh Gandhe <1909520+san81@users.noreply.github.com> Date: Fri, 31 Jan 2025 12:18:38 -0800 Subject: [PATCH 02/14] matching config structure to be more inline for Confluence Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com> --- .../source/confluence/ConfluenceService.java | 13 +------ .../configuration/FilterConfig.java | 11 ++---- .../configuration/IssueTypeConfig.java | 29 -------------- .../confluence/configuration/NameConfig.java | 4 +- ...{StatusConfig.java => PageTypeConfig.java} | 6 +-- .../{ProjectConfig.java => SpaceConfig.java} | 2 +- .../utils/ConfluenceConfigHelper.java | 38 +++++-------------- .../ConfluenceConfigHelperTest.java | 34 +++++------------ .../ConfluenceSourceConfigTest.java | 4 +- 9 files changed, 33 insertions(+), 108 deletions(-) delete mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/IssueTypeConfig.java rename data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/{StatusConfig.java => PageTypeConfig.java} (77%) rename data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/{ProjectConfig.java => SpaceConfig.java} (94%) diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java index e1dfe6e481..81ddd8ce12 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java @@ -41,8 +41,6 @@ import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.PREFIX; import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.PROJECT_IN; import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.PROJECT_NOT_IN; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.STATUS_IN; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.STATUS_NOT_IN; import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.SUFFIX; @@ -156,16 +154,7 @@ private StringBuilder createIssueFilterCriteria(ConfluenceSourceConfig configura .collect(Collectors.joining(DELIMITER, PREFIX, SUFFIX))) .append(CLOSING_ROUND_BRACKET); } - if (!CollectionUtils.isEmpty(ConfluenceConfigHelper.getIssueStatusIncludeFilter(configuration))) { - jiraQl.append(STATUS_IN).append(ConfluenceConfigHelper.getIssueStatusIncludeFilter(configuration).stream() - .collect(Collectors.joining(DELIMITER, PREFIX, SUFFIX))) - .append(CLOSING_ROUND_BRACKET); - } - if (!CollectionUtils.isEmpty(ConfluenceConfigHelper.getIssueStatusExcludeFilter(configuration))) { - jiraQl.append(STATUS_NOT_IN).append(ConfluenceConfigHelper.getIssueStatusExcludeFilter(configuration).stream() - .collect(Collectors.joining(DELIMITER, PREFIX, SUFFIX))) - .append(CLOSING_ROUND_BRACKET); - } + log.error("Created issue filter criteria JiraQl query: {}", jiraQl); return jiraQl; } diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/FilterConfig.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/FilterConfig.java index 040ce85107..0ec468f2d8 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/FilterConfig.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/FilterConfig.java @@ -15,12 +15,9 @@ @Getter public class FilterConfig { - @JsonProperty("project") - private ProjectConfig projectConfig; + @JsonProperty("space") + private SpaceConfig spaceConfig; - @JsonProperty("status") - private StatusConfig statusConfig; - - @JsonProperty("issue_type") - private IssueTypeConfig issueTypeConfig; + @JsonProperty("page_type") + private PageTypeConfig pageTypeConfig; } diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/IssueTypeConfig.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/IssueTypeConfig.java deleted file mode 100644 index 56b9ff9208..0000000000 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/IssueTypeConfig.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - */ - -package org.opensearch.dataprepper.plugins.source.confluence.configuration; - -import com.fasterxml.jackson.annotation.JsonProperty; -import jakarta.validation.constraints.Size; -import lombok.Getter; - -import java.util.ArrayList; -import java.util.List; - -@Getter -public class IssueTypeConfig { - @JsonProperty("include") - @Size(max = 1000, message = "Issue type filter should not be more than 1000") - private List include = new ArrayList<>(); - - @JsonProperty("exclude") - @Size(max = 1000, message = "Issue type filter should not be more than 1000") - private List exclude = new ArrayList<>(); -} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/NameConfig.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/NameConfig.java index 8982d69cf2..8e3a513d56 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/NameConfig.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/NameConfig.java @@ -21,10 +21,10 @@ @Getter public class NameConfig { @JsonProperty("include") - @Size(max = 1000, message = "Project name type filter should not be more than 1000") + @Size(max = 1000, message = "Space name type filter should not be more than 1000") private List include = new ArrayList<>(); @JsonProperty("exclude") - @Size(max = 1000, message = "Project name type filter should not be more than 1000") + @Size(max = 1000, message = "Space name type filter should not be more than 1000") private List exclude = new ArrayList<>(); } diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/StatusConfig.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/PageTypeConfig.java similarity index 77% rename from data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/StatusConfig.java rename to data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/PageTypeConfig.java index 3dde46bfd9..756e9ac91b 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/StatusConfig.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/PageTypeConfig.java @@ -18,12 +18,12 @@ import java.util.List; @Getter -public class StatusConfig { +public class PageTypeConfig { @JsonProperty("include") - @Size(max = 1000, message = "status type filter should not be more than 1000") + @Size(max = 1000, message = "Page type filter should not be more than 1000") private List include = new ArrayList<>(); @JsonProperty("exclude") - @Size(max = 1000, message = "status type filter should not be more than 1000") + @Size(max = 1000, message = "Page type filter should not be more than 1000") private List exclude = new ArrayList<>(); } diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/ProjectConfig.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/SpaceConfig.java similarity index 94% rename from data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/ProjectConfig.java rename to data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/SpaceConfig.java index 631c7ae7bf..fe572011ff 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/ProjectConfig.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/SpaceConfig.java @@ -14,7 +14,7 @@ import lombok.Getter; @Getter -public class ProjectConfig { +public class SpaceConfig { @JsonProperty("key") private NameConfig nameConfig; } diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/ConfluenceConfigHelper.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/ConfluenceConfigHelper.java index fa31c16955..09e334eea0 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/ConfluenceConfigHelper.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/ConfluenceConfigHelper.java @@ -25,24 +25,6 @@ */ @Slf4j public class ConfluenceConfigHelper { - /** - * Get Issue Status Filter from repository configuration. - * - * @return List Issue Status Filter. - */ - public static List getIssueStatusIncludeFilter(ConfluenceSourceConfig repositoryConfiguration) { - if (repositoryConfiguration.getFilterConfig() == null || repositoryConfiguration.getFilterConfig().getStatusConfig() == null) { - return new ArrayList<>(); - } - return repositoryConfiguration.getFilterConfig().getStatusConfig().getInclude(); - } - - public static List getIssueStatusExcludeFilter(ConfluenceSourceConfig repositoryConfiguration) { - if (repositoryConfiguration.getFilterConfig() == null || repositoryConfiguration.getFilterConfig().getStatusConfig() == null) { - return new ArrayList<>(); - } - return repositoryConfiguration.getFilterConfig().getStatusConfig().getExclude(); - } /** * Get Issue Types Filter from repository configuration. @@ -50,17 +32,17 @@ public static List getIssueStatusExcludeFilter(ConfluenceSourceConfig re * @return List Issue Type Filter. */ public static List getIssueTypeIncludeFilter(ConfluenceSourceConfig repositoryConfiguration) { - if (repositoryConfiguration.getFilterConfig() == null || repositoryConfiguration.getFilterConfig().getIssueTypeConfig() == null) { + if (repositoryConfiguration.getFilterConfig() == null || repositoryConfiguration.getFilterConfig().getPageTypeConfig() == null) { return new ArrayList<>(); } - return repositoryConfiguration.getFilterConfig().getIssueTypeConfig().getInclude(); + return repositoryConfiguration.getFilterConfig().getPageTypeConfig().getInclude(); } public static List getIssueTypeExcludeFilter(ConfluenceSourceConfig repositoryConfiguration) { - if (repositoryConfiguration.getFilterConfig() == null || repositoryConfiguration.getFilterConfig().getIssueTypeConfig() == null) { + if (repositoryConfiguration.getFilterConfig() == null || repositoryConfiguration.getFilterConfig().getPageTypeConfig() == null) { return new ArrayList<>(); } - return repositoryConfiguration.getFilterConfig().getIssueTypeConfig().getExclude(); + return repositoryConfiguration.getFilterConfig().getPageTypeConfig().getExclude(); } /** @@ -71,20 +53,20 @@ public static List getIssueTypeExcludeFilter(ConfluenceSourceConfig repo */ public static List getProjectNameIncludeFilter(ConfluenceSourceConfig repositoryConfiguration) { if (repositoryConfiguration.getFilterConfig() == null || - repositoryConfiguration.getFilterConfig().getProjectConfig() == null || - repositoryConfiguration.getFilterConfig().getProjectConfig().getNameConfig() == null) { + repositoryConfiguration.getFilterConfig().getSpaceConfig() == null || + repositoryConfiguration.getFilterConfig().getSpaceConfig().getNameConfig() == null) { return new ArrayList<>(); } - return repositoryConfiguration.getFilterConfig().getProjectConfig().getNameConfig().getInclude(); + return repositoryConfiguration.getFilterConfig().getSpaceConfig().getNameConfig().getInclude(); } public static List getProjectNameExcludeFilter(ConfluenceSourceConfig repositoryConfiguration) { if (repositoryConfiguration.getFilterConfig() == null || - repositoryConfiguration.getFilterConfig().getProjectConfig() == null || - repositoryConfiguration.getFilterConfig().getProjectConfig().getNameConfig() == null) { + repositoryConfiguration.getFilterConfig().getSpaceConfig() == null || + repositoryConfiguration.getFilterConfig().getSpaceConfig().getNameConfig() == null) { return new ArrayList<>(); } - return repositoryConfiguration.getFilterConfig().getProjectConfig().getNameConfig().getExclude(); + return repositoryConfiguration.getFilterConfig().getSpaceConfig().getNameConfig().getExclude(); } diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceConfigHelperTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceConfigHelperTest.java index 4bddc9d839..d83967cf0b 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceConfigHelperTest.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceConfigHelperTest.java @@ -18,11 +18,10 @@ import org.opensearch.dataprepper.plugins.source.confluence.configuration.AuthenticationConfig; import org.opensearch.dataprepper.plugins.source.confluence.configuration.BasicConfig; import org.opensearch.dataprepper.plugins.source.confluence.configuration.FilterConfig; -import org.opensearch.dataprepper.plugins.source.confluence.configuration.IssueTypeConfig; import org.opensearch.dataprepper.plugins.source.confluence.configuration.NameConfig; import org.opensearch.dataprepper.plugins.source.confluence.configuration.Oauth2Config; -import org.opensearch.dataprepper.plugins.source.confluence.configuration.ProjectConfig; -import org.opensearch.dataprepper.plugins.source.confluence.configuration.StatusConfig; +import org.opensearch.dataprepper.plugins.source.confluence.configuration.PageTypeConfig; +import org.opensearch.dataprepper.plugins.source.confluence.configuration.SpaceConfig; import org.opensearch.dataprepper.plugins.source.confluence.utils.ConfluenceConfigHelper; import java.util.List; @@ -49,10 +48,10 @@ public class ConfluenceConfigHelperTest { StatusConfig statusConfig; @Mock - IssueTypeConfig issueTypeConfig; + PageTypeConfig pageTypeConfig; @Mock - ProjectConfig projectConfig; + SpaceConfig spaceConfig; @Mock NameConfig nameConfig; @@ -77,31 +76,18 @@ void testInitialization() { ConfluenceConfigHelper confluenceConfigHelper = new ConfluenceConfigHelper(); assertNotNull(confluenceConfigHelper); } - - @Test - void testGetIssueStatusFilter() { - when(confluenceSourceConfig.getFilterConfig()).thenReturn(filterConfig); - when(filterConfig.getStatusConfig()).thenReturn(statusConfig); - assertTrue(ConfluenceConfigHelper.getIssueStatusIncludeFilter(confluenceSourceConfig).isEmpty()); - assertTrue(ConfluenceConfigHelper.getIssueStatusExcludeFilter(confluenceSourceConfig).isEmpty()); - List issueStatusFilter = List.of("Done", "In Progress"); - List issueStatusExcludeFilter = List.of("Done2", "In Progress2"); - when(statusConfig.getInclude()).thenReturn(issueStatusFilter); - when(statusConfig.getExclude()).thenReturn(issueStatusExcludeFilter); - assertEquals(issueStatusFilter, ConfluenceConfigHelper.getIssueStatusIncludeFilter(confluenceSourceConfig)); - assertEquals(issueStatusExcludeFilter, ConfluenceConfigHelper.getIssueStatusExcludeFilter(confluenceSourceConfig)); - } + @Test void testGetIssueTypeFilter() { when(confluenceSourceConfig.getFilterConfig()).thenReturn(filterConfig); - when(filterConfig.getIssueTypeConfig()).thenReturn(issueTypeConfig); + when(filterConfig.getPageTypeConfig()).thenReturn(pageTypeConfig); assertTrue(ConfluenceConfigHelper.getIssueTypeIncludeFilter(confluenceSourceConfig).isEmpty()); assertTrue(ConfluenceConfigHelper.getIssueTypeExcludeFilter(confluenceSourceConfig).isEmpty()); List issueTypeFilter = List.of("Bug", "Story"); List issueTypeExcludeFilter = List.of("Bug2", "Story2"); - when(issueTypeConfig.getInclude()).thenReturn(issueTypeFilter); - when(issueTypeConfig.getExclude()).thenReturn(issueTypeExcludeFilter); + when(pageTypeConfig.getInclude()).thenReturn(issueTypeFilter); + when(pageTypeConfig.getExclude()).thenReturn(issueTypeExcludeFilter); assertEquals(issueTypeFilter, ConfluenceConfigHelper.getIssueTypeIncludeFilter(confluenceSourceConfig)); assertEquals(issueTypeExcludeFilter, ConfluenceConfigHelper.getIssueTypeExcludeFilter(confluenceSourceConfig)); } @@ -109,8 +95,8 @@ void testGetIssueTypeFilter() { @Test void testGetProjectNameFilter() { when(confluenceSourceConfig.getFilterConfig()).thenReturn(filterConfig); - when(filterConfig.getProjectConfig()).thenReturn(projectConfig); - when(projectConfig.getNameConfig()).thenReturn(nameConfig); + when(filterConfig.getSpaceConfig()).thenReturn(spaceConfig); + when(spaceConfig.getNameConfig()).thenReturn(nameConfig); assertTrue(ConfluenceConfigHelper.getProjectNameIncludeFilter(confluenceSourceConfig).isEmpty()); assertTrue(ConfluenceConfigHelper.getProjectNameExcludeFilter(confluenceSourceConfig).isEmpty()); List projectNameFilter = List.of("TEST", "TEST2"); diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceConfigTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceConfigTest.java index d3bbb7411b..ddf84b8004 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceConfigTest.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceConfigTest.java @@ -112,8 +112,8 @@ private ConfluenceSourceConfig createJiraSourceConfig(String authtype, boolean h @Test void testGetters() throws Exception { confluenceSourceConfig = createJiraSourceConfig(BASIC, false); - assertEquals(confluenceSourceConfig.getFilterConfig().getIssueTypeConfig().getInclude(), issueTypeList); - assertEquals(confluenceSourceConfig.getFilterConfig().getProjectConfig().getNameConfig().getInclude(), projectList); + assertEquals(confluenceSourceConfig.getFilterConfig().getPageTypeConfig().getInclude(), issueTypeList); + assertEquals(confluenceSourceConfig.getFilterConfig().getSpaceConfig().getNameConfig().getInclude(), projectList); assertEquals(confluenceSourceConfig.getFilterConfig().getStatusConfig().getInclude(), statusList); assertEquals(confluenceSourceConfig.getAccountUrl(), accountUrl); assertEquals(confluenceSourceConfig.getAuthenticationConfig().getBasicConfig().getPassword(), password); From 6f40d6592ce34cb371ceddb66160a26c43995e03 Mon Sep 17 00:00:00 2001 From: Santhosh Gandhe <1909520+san81@users.noreply.github.com> Date: Fri, 31 Jan 2025 19:24:32 -0800 Subject: [PATCH 03/14] Functional confluence version Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com> --- .../source/confluence/ConfluenceClient.java | 12 +- .../source/confluence/ConfluenceItemInfo.java | 68 ++++----- .../source/confluence/ConfluenceIterator.java | 2 +- .../source/confluence/ConfluenceService.java | 99 ++++++------- .../confluence/models/ConfluenceItem.java | 65 +++++++++ ...ults.java => ConfluenceSearchResults.java} | 23 +-- .../source/confluence/models/IssueBean.java | 125 ----------------- .../source/confluence/models/SpaceItem.java | 61 ++++++++ .../confluence/rest/ConfluenceRestClient.java | 63 ++++----- .../rest/auth/ConfluenceOauthConfig.java | 2 +- .../utils/ConfluenceConfigHelper.java | 21 ++- .../source/confluence/utils/Constants.java | 12 +- .../{JqlConstants.java => CqlConstants.java} | 16 +-- .../confluence/ConfluenceClientTest.java | 10 +- .../ConfluenceConfigHelperTest.java | 21 ++- .../confluence/ConfluenceItemInfoTest.java | 17 ++- .../confluence/ConfluenceIteratorTest.java | 113 +-------------- .../confluence/ConfluenceServiceTest.java | 67 ++++----- .../ConfluenceSourceConfigTest.java | 1 - .../confluence/models/ConfluenceItemTest.java | 71 ++++++++++ .../models/ConfluenceSearchResultsTest.java | 41 ++++++ .../confluence/models/IssueBeanTest.java | 131 ------------------ .../confluence/models/SearchResultsTest.java | 98 ------------- .../rest/ConfluenceRestClientTest.java | 41 +++--- .../utils/AddressValidationTest.java | 5 +- 25 files changed, 472 insertions(+), 713 deletions(-) create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceItem.java rename data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/{SearchResults.java => ConfluenceSearchResults.java} (53%) delete mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/IssueBean.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/SpaceItem.java rename data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/{JqlConstants.java => CqlConstants.java} (55%) create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceItemTest.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceSearchResultsTest.java delete mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/IssueBeanTest.java delete mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/SearchResultsTest.java diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceClient.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceClient.java index 68874f8587..f4dc7cfe4f 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceClient.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceClient.java @@ -41,7 +41,7 @@ import java.util.stream.Collectors; import static java.util.concurrent.CompletableFuture.supplyAsync; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.PROJECT; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.SPACE; /** * This class represents a Jira client. @@ -93,7 +93,7 @@ public void executePartition(SaasWorkerProgressState state, state.getKeyAttributes(), state.getItemIds().size()); List itemIds = state.getItemIds(); Map keyAttributes = state.getKeyAttributes(); - String project = (String) keyAttributes.get(PROJECT); + String project = (String) keyAttributes.get(SPACE); Instant eventTime = state.getExportStartTime(); List itemInfos = new ArrayList<>(); for (String itemId : itemIds) { @@ -103,7 +103,7 @@ public void executePartition(SaasWorkerProgressState state, ItemInfo itemInfo = ConfluenceItemInfo.builder() .withItemId(itemId) .withId(itemId) - .withProject(project) + .withSpace(project) .withEventTime(eventTime) .withMetadata(keyAttributes).build(); itemInfos.add(itemInfo); @@ -112,12 +112,12 @@ public void executePartition(SaasWorkerProgressState state, String eventType = EventType.DOCUMENT.toString(); List> recordsToWrite = itemInfos .parallelStream() - .map(t -> (Supplier) (() -> service.getIssue(t.getId()))) + .map(t -> (Supplier) (() -> service.getContent(t.getId()))) .map(supplier -> supplyAsync(supplier, this.executorService)) .map(CompletableFuture::join) - .map(ticketJson -> { + .map(contentJson -> { try { - return objectMapper.readValue(ticketJson, new TypeReference<>() { + return objectMapper.readValue(contentJson, new TypeReference<>() { }); } catch (JsonProcessingException e) { throw new RuntimeException(e); diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceItemInfo.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceItemInfo.java index 15aa46009c..c6048caf65 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceItemInfo.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceItemInfo.java @@ -12,7 +12,7 @@ import lombok.Getter; import lombok.Setter; -import org.opensearch.dataprepper.plugins.source.confluence.models.IssueBean; +import org.opensearch.dataprepper.plugins.source.confluence.models.ConfluenceItem; import org.opensearch.dataprepper.plugins.source.confluence.utils.ConfluenceContentType; import org.opensearch.dataprepper.plugins.source.confluence.utils.Constants; import org.opensearch.dataprepper.plugins.source.source_crawler.model.ItemInfo; @@ -22,18 +22,18 @@ import java.util.Map; import java.util.UUID; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.CONTENT_ID; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.CONTENT_TITLE; import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.CREATED; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.ISSUE_KEY; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.PROJECT_KEY; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.PROJECT_NAME; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.UPDATED; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants._ISSUE; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.LAST_MODIFIED; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.SPACE_KEY; @Setter @Getter public class ConfluenceItemInfo implements ItemInfo { - private String project; - private String issueType; + private String space; + // either a page or a BlogPost + private String contentType; private String id; private String itemId; private Map metadata; @@ -41,14 +41,14 @@ public class ConfluenceItemInfo implements ItemInfo { public ConfluenceItemInfo(String id, String itemId, - String project, - String issueType, + String space, + String contentType, Map metadata, Instant eventTime ) { this.id = id; - this.project = project; - this.issueType = issueType; + this.space = space; + this.contentType = contentType; this.itemId = itemId; this.metadata = metadata; this.eventTime = eventTime; @@ -60,7 +60,7 @@ public static ConfluenceItemInfoBuilder builder() { @Override public String getPartitionKey() { - return project + "|" + issueType + "|" + UUID.randomUUID(); + return space + "|" + contentType + "|" + UUID.randomUUID(); } @Override @@ -70,12 +70,12 @@ public String getId() { @Override public Map getKeyAttributes() { - return Map.of(Constants.PROJECT, project); + return Map.of(Constants.SPACE, space); } @Override public Instant getLastModifiedAt() { - long updatedAtMillis = getMetadataField(Constants.UPDATED); + long updatedAtMillis = getMetadataField(Constants.LAST_MODIFIED); long createdAtMillis = getMetadataField(Constants.CREATED); return createdAtMillis > updatedAtMillis ? Instant.ofEpochMilli(createdAtMillis) : Instant.ofEpochMilli(updatedAtMillis); @@ -102,14 +102,14 @@ public static class ConfluenceItemInfoBuilder { private Instant eventTime; private String id; private String itemId; - private String project; - private String issueType; + private String space; + private String contentType; public ConfluenceItemInfoBuilder() { } public ConfluenceItemInfo build() { - return new ConfluenceItemInfo(id, itemId, project, issueType, metadata, eventTime); + return new ConfluenceItemInfo(id, itemId, space, contentType, metadata, eventTime); } public ConfluenceItemInfoBuilder withMetadata(Map metadata) { @@ -132,25 +132,25 @@ public ConfluenceItemInfoBuilder withId(String id) { return this; } - public ConfluenceItemInfoBuilder withProject(String project) { - this.project = project; + public ConfluenceItemInfoBuilder withSpace(String space) { + this.space = space; return this; } - public ConfluenceItemInfoBuilder withIssueBean(IssueBean issue) { - Map issueMetadata = new HashMap<>(); - issueMetadata.put(PROJECT_KEY, issue.getProject()); - issueMetadata.put(PROJECT_NAME, issue.getProjectName()); - issueMetadata.put(CREATED, issue.getCreatedTimeMillis()); - issueMetadata.put(UPDATED, issue.getUpdatedTimeMillis()); - issueMetadata.put(ISSUE_KEY, issue.getKey()); - issueMetadata.put(ConfluenceService.CONTENT_TYPE, ConfluenceContentType.ISSUE.getType()); - - this.project = issue.getProject(); - this.id = issue.getKey(); - this.issueType = ConfluenceContentType.ISSUE.getType(); - this.itemId = _ISSUE + issueMetadata.get(PROJECT_KEY) + "-" + issue.getKey(); - this.metadata = issueMetadata; + public ConfluenceItemInfoBuilder withIssueBean(ConfluenceItem contentItem) { + Map contentItemMetadata = new HashMap<>(); + contentItemMetadata.put(SPACE_KEY, contentItem.getSpaceItem().getKey()); + contentItemMetadata.put(CONTENT_TITLE, contentItem.getTitle()); + contentItemMetadata.put(CREATED, contentItem.getCreatedTimeMillis()); + contentItemMetadata.put(LAST_MODIFIED, contentItem.getUpdatedTimeMillis()); + contentItemMetadata.put(CONTENT_ID, contentItem.getId()); + contentItemMetadata.put(ConfluenceService.CONTENT_TYPE, ConfluenceContentType.ISSUE.getType()); + + this.space = contentItem.getSpaceItem().getKey(); + this.id = contentItem.getId(); + this.contentType = ConfluenceContentType.ISSUE.getType(); + this.itemId = contentItem.getId(); + this.metadata = contentItemMetadata; return this; } } diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceIterator.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceIterator.java index bc67568083..fb746c06ca 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceIterator.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceIterator.java @@ -88,7 +88,7 @@ private boolean isCrawlerRunning() { private void startCrawlerThreads() { futureList.add(crawlerTaskExecutor.submit(() -> - service.getJiraEntities(sourceConfig, lastPollTime, itemInfoQueue), false)); + service.getPages(sourceConfig, lastPollTime, itemInfoQueue), false)); } @Override diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java index 81ddd8ce12..457c98c40c 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java @@ -14,8 +14,8 @@ import lombok.extern.slf4j.Slf4j; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.plugins.source.confluence.exception.BadRequestException; -import org.opensearch.dataprepper.plugins.source.confluence.models.IssueBean; -import org.opensearch.dataprepper.plugins.source.confluence.models.SearchResults; +import org.opensearch.dataprepper.plugins.source.confluence.models.ConfluenceItem; +import org.opensearch.dataprepper.plugins.source.confluence.models.ConfluenceSearchResults; import org.opensearch.dataprepper.plugins.source.confluence.rest.ConfluenceRestClient; import org.opensearch.dataprepper.plugins.source.confluence.utils.ConfluenceConfigHelper; import org.opensearch.dataprepper.plugins.source.source_crawler.model.ItemInfo; @@ -23,6 +23,9 @@ import javax.inject.Named; import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -32,16 +35,16 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.UPDATED; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.CLOSING_ROUND_BRACKET; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.DELIMITER; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.GREATER_THAN_EQUALS; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.ISSUE_TYPE_IN; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.ISSUE_TYPE_NOT_IN; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.PREFIX; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.PROJECT_IN; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.PROJECT_NOT_IN; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.SUFFIX; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.LAST_MODIFIED; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.CqlConstants.CLOSING_ROUND_BRACKET; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.CqlConstants.CONTENT_TYPE_IN; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.CqlConstants.CONTENT_TYPE_NOT_IN; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.CqlConstants.DELIMITER; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.CqlConstants.GREATER_THAN_EQUALS; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.CqlConstants.PREFIX; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.CqlConstants.SPACE_IN; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.CqlConstants.SPACE_NOT_IN; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.CqlConstants.SUFFIX; /** @@ -74,14 +77,14 @@ public ConfluenceService(ConfluenceSourceConfig confluenceSourceConfig, Confluen * @param configuration the configuration. * @param timestamp timestamp. */ - public void getJiraEntities(ConfluenceSourceConfig configuration, Instant timestamp, Queue itemInfoQueue) { + public void getPages(ConfluenceSourceConfig configuration, Instant timestamp, Queue itemInfoQueue) { log.trace("Started to fetch entities"); - searchForNewTicketsAndAddToQueue(configuration, timestamp, itemInfoQueue); + searchForNewContentAndAddToQueue(configuration, timestamp, itemInfoQueue); log.trace("Creating item information and adding in queue"); } - public String getIssue(String issueKey) { - return confluenceRestClient.getIssue(issueKey); + public String getContent(String contentId) { + return confluenceRestClient.getContent(contentId); } /** @@ -90,17 +93,17 @@ public String getIssue(String issueKey) { * @param configuration Input Parameter * @param timestamp Input Parameter */ - private void searchForNewTicketsAndAddToQueue(ConfluenceSourceConfig configuration, Instant timestamp, + private void searchForNewContentAndAddToQueue(ConfluenceSourceConfig configuration, Instant timestamp, Queue itemInfoQueue) { log.trace("Looking for Add/Modified tickets with a Search API call"); - StringBuilder jql = createIssueFilterCriteria(configuration, timestamp); + StringBuilder cql = createContentFilterCriteria(configuration, timestamp); int total; int startAt = 0; do { - SearchResults searchIssues = confluenceRestClient.getAllIssues(jql, startAt, configuration); - List issueList = new ArrayList<>(searchIssues.getIssues()); - total = searchIssues.getTotal(); - startAt += searchIssues.getIssues().size(); + ConfluenceSearchResults searchIssues = confluenceRestClient.getAllContent(cql, startAt); + List issueList = new ArrayList<>(searchIssues.getResults()); + total = searchIssues.getSize(); + startAt += searchIssues.getResults().size(); addItemsToQueue(issueList, itemInfoQueue); } while (startAt < total); searchResultsFoundCounter.increment(total); @@ -113,7 +116,7 @@ private void searchForNewTicketsAndAddToQueue(ConfluenceSourceConfig configurati * @param issueList Issue list. * @param itemInfoQueue Item info queue. */ - private void addItemsToQueue(List issueList, Queue itemInfoQueue) { + private void addItemsToQueue(List issueList, Queue itemInfoQueue) { issueList.forEach(issue -> { itemInfoQueue.add(ConfluenceItemInfo.builder().withEventTime(Instant.now()).withIssueBean(issue).build()); }); @@ -121,63 +124,65 @@ private void addItemsToQueue(List issueList, Queue itemInfo /** - * Method for creating Issue Filter Criteria. + * Method for creating Content Filter Criteria. * * @param configuration Input Parameter * @param ts Input Parameter * @return String Builder */ - private StringBuilder createIssueFilterCriteria(ConfluenceSourceConfig configuration, Instant ts) { + private StringBuilder createContentFilterCriteria(ConfluenceSourceConfig configuration, Instant ts) { - log.info("Creating issue filter criteria"); - if (!CollectionUtils.isEmpty(ConfluenceConfigHelper.getProjectNameIncludeFilter(configuration)) || !CollectionUtils.isEmpty(ConfluenceConfigHelper.getProjectNameExcludeFilter(configuration))) { - validateProjectFilters(configuration); + log.info("Creating content filter criteria"); + if (!CollectionUtils.isEmpty(ConfluenceConfigHelper.getSpacesNameIncludeFilter(configuration)) || !CollectionUtils.isEmpty(ConfluenceConfigHelper.getSpacesNameExcludeFilter(configuration))) { + validateSpaceFilters(configuration); } - StringBuilder jiraQl = new StringBuilder(UPDATED + GREATER_THAN_EQUALS + ts.toEpochMilli()); - if (!CollectionUtils.isEmpty(ConfluenceConfigHelper.getProjectNameIncludeFilter(configuration))) { - jiraQl.append(PROJECT_IN).append(ConfluenceConfigHelper.getProjectNameIncludeFilter(configuration).stream() + String formattedTimeStamp = LocalDateTime.ofInstant(ts, ZoneId.systemDefault()) + .format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm")); + StringBuilder cQl = new StringBuilder(LAST_MODIFIED + GREATER_THAN_EQUALS + "\"" + formattedTimeStamp + "\""); + if (!CollectionUtils.isEmpty(ConfluenceConfigHelper.getSpacesNameIncludeFilter(configuration))) { + cQl.append(SPACE_IN).append(ConfluenceConfigHelper.getSpacesNameIncludeFilter(configuration).stream() .collect(Collectors.joining(DELIMITER, PREFIX, SUFFIX))) .append(CLOSING_ROUND_BRACKET); } - if (!CollectionUtils.isEmpty(ConfluenceConfigHelper.getProjectNameExcludeFilter(configuration))) { - jiraQl.append(PROJECT_NOT_IN).append(ConfluenceConfigHelper.getProjectNameExcludeFilter(configuration).stream() + if (!CollectionUtils.isEmpty(ConfluenceConfigHelper.getSpacesNameExcludeFilter(configuration))) { + cQl.append(SPACE_NOT_IN).append(ConfluenceConfigHelper.getSpacesNameExcludeFilter(configuration).stream() .collect(Collectors.joining(DELIMITER, PREFIX, SUFFIX))) .append(CLOSING_ROUND_BRACKET); } - if (!CollectionUtils.isEmpty(ConfluenceConfigHelper.getIssueTypeIncludeFilter(configuration))) { - jiraQl.append(ISSUE_TYPE_IN).append(ConfluenceConfigHelper.getIssueTypeIncludeFilter(configuration).stream() + if (!CollectionUtils.isEmpty(ConfluenceConfigHelper.getContentTypeIncludeFilter(configuration))) { + cQl.append(CONTENT_TYPE_IN).append(ConfluenceConfigHelper.getContentTypeIncludeFilter(configuration).stream() .collect(Collectors.joining(DELIMITER, PREFIX, SUFFIX))) .append(CLOSING_ROUND_BRACKET); } - if (!CollectionUtils.isEmpty(ConfluenceConfigHelper.getIssueTypeExcludeFilter(configuration))) { - jiraQl.append(ISSUE_TYPE_NOT_IN).append(ConfluenceConfigHelper.getIssueTypeExcludeFilter(configuration).stream() + if (!CollectionUtils.isEmpty(ConfluenceConfigHelper.getContentTypeExcludeFilter(configuration))) { + cQl.append(CONTENT_TYPE_NOT_IN).append(ConfluenceConfigHelper.getContentTypeExcludeFilter(configuration).stream() .collect(Collectors.joining(DELIMITER, PREFIX, SUFFIX))) .append(CLOSING_ROUND_BRACKET); } - log.error("Created issue filter criteria JiraQl query: {}", jiraQl); - return jiraQl; + log.error("Created issue filter criteria JiraQl query: {}", cQl); + return cQl; } /** - * Method for Validating Project Filters. + * Method for Validating Space Filters. * * @param configuration Input Parameter */ - private void validateProjectFilters(ConfluenceSourceConfig configuration) { + private void validateSpaceFilters(ConfluenceSourceConfig configuration) { log.trace("Validating project filters"); List badFilters = new ArrayList<>(); Set includedProjects = new HashSet<>(); List includedAndExcludedProjects = new ArrayList<>(); Pattern regex = Pattern.compile("[^A-Z0-9]"); - ConfluenceConfigHelper.getProjectNameIncludeFilter(configuration).forEach(projectFilter -> { + ConfluenceConfigHelper.getSpacesNameIncludeFilter(configuration).forEach(projectFilter -> { Matcher matcher = regex.matcher(projectFilter); includedProjects.add(projectFilter); if (matcher.find() || projectFilter.length() <= 1 || projectFilter.length() > 10) { badFilters.add(projectFilter); } }); - ConfluenceConfigHelper.getProjectNameExcludeFilter(configuration).forEach(projectFilter -> { + ConfluenceConfigHelper.getSpacesNameExcludeFilter(configuration).forEach(projectFilter -> { Matcher matcher = regex.matcher(projectFilter); if (includedProjects.contains(projectFilter)) { includedAndExcludedProjects.add(projectFilter); @@ -188,16 +193,16 @@ private void validateProjectFilters(ConfluenceSourceConfig configuration) { }); if (!badFilters.isEmpty()) { String filters = String.join("\"" + badFilters + "\"", ", "); - log.error("One or more invalid project keys found in filter configuration: {}", badFilters); + log.error("One or more invalid Space keys found in filter configuration: {}", badFilters); throw new BadRequestException("Bad request exception occurred " + - "Invalid project key found in filter configuration for " + "Invalid Space key found in filter configuration for " + filters); } if (!includedAndExcludedProjects.isEmpty()) { String filters = String.join("\"" + includedAndExcludedProjects + "\"", ", "); - log.error("One or more project keys found in both include and exclude: {}", includedAndExcludedProjects); + log.error("One or more Space keys found in both include and exclude: {}", includedAndExcludedProjects); throw new BadRequestException("Bad request exception occurred " + - "Project filters is invalid because the following projects are listed in both include and exclude" + "Space filters is invalid because the following space are listed in both include and exclude" + filters); } diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceItem.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceItem.java new file mode 100644 index 0000000000..1806627793 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceItem.java @@ -0,0 +1,65 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.models; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Getter; +import lombok.Setter; + + +@Setter +@Getter +@JsonIgnoreProperties(ignoreUnknown = true) +public class ConfluenceItem { + + /** + * The ID of the issue. + */ + @JsonProperty("id") + private String id = null; + + /** + * The type of the issue. + */ + @JsonProperty("type") + private String type = null; + + /** + * The type of the issue. + */ + @JsonProperty("status") + private String status = null; + + /** + * The type of the issue. + */ + @JsonProperty("title") + private String title = null; + + /** + * Space this content belongs to + */ + @JsonProperty("space") + private SpaceItem spaceItem; + + @JsonIgnore + public long getCreatedTimeMillis() { + return 0L; + } + + @JsonIgnore + public long getUpdatedTimeMillis() { + return 0L; + } + +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/SearchResults.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceSearchResults.java similarity index 53% rename from data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/SearchResults.java rename to data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceSearchResults.java index 17fd06803c..d56ca84305 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/SearchResults.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceSearchResults.java @@ -11,6 +11,7 @@ package org.opensearch.dataprepper.plugins.source.confluence.models; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Getter; @@ -20,20 +21,20 @@ * The result of a JQL search. */ @Getter -public class SearchResults { - @JsonProperty("expand") - private String expand = null; +@JsonIgnoreProperties(ignoreUnknown = true) +public class ConfluenceSearchResults { - @JsonProperty("startAt") - private Integer startAt = null; - @JsonProperty("maxResults") - private Integer maxResults = null; + @JsonProperty("start") + private Integer start = null; - @JsonProperty("total") - private Integer total = null; + @JsonProperty("limit") + private Integer limit = null; - @JsonProperty("issues") - private List issues = null; + @JsonProperty("size") + private Integer size = null; + + @JsonProperty("results") + private List results = null; } diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/IssueBean.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/IssueBean.java deleted file mode 100644 index 57b6f49774..0000000000 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/IssueBean.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - */ - -package org.opensearch.dataprepper.plugins.source.confluence.models; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Getter; -import lombok.Setter; - -import java.time.OffsetDateTime; -import java.time.format.DateTimeFormatter; -import java.util.Date; -import java.util.Map; -import java.util.Objects; -import java.util.regex.Pattern; - -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.CREATED; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.KEY; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.NAME; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.PROJECT; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.UPDATED; - - -public class IssueBean { - - /** - * Expand options that include additional issue details in the response. - */ - @Getter - @Setter - @JsonProperty("expand") - private String expand = null; - /** - * The ID of the issue. - */ - @Getter - @Setter - @JsonProperty("id") - private String id = null; - /** - * The URL of the issue details. - */ - @Getter - @Setter - @JsonProperty("self") - private String self = null; - /** - * The key of the issue. - */ - @Getter - @Setter - @JsonProperty("key") - private String key = null; - @Getter - @Setter - @JsonProperty("renderedFields") - private Map renderedFields = null; - @Getter - @Setter - @JsonProperty("properties") - private Map properties = null; - @Getter - @Setter - @JsonProperty("names") - private Map names = null; - @Getter - @Setter - @JsonProperty("fields") - private Map fields = null; - @JsonIgnore - private final Pattern JiraDateTimePattern = Pattern.compile( - "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}[-+]\\d{4}$"); - @JsonIgnore - private final DateTimeFormatter offsetDateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ"); - - @JsonIgnore - public String getProject() { - if (fields != null && Objects.nonNull(((Map) fields.get(PROJECT)).get(KEY))) { - return ((Map) fields.get(PROJECT)).get(KEY).toString(); - } - return null; - } - - @JsonIgnore - public String getProjectName() { - if (fields != null && Objects.nonNull(((Map) fields.get(PROJECT)).get(NAME))) { - return ((Map) fields.get(PROJECT)).get(NAME).toString(); - } - return null; - } - - @JsonIgnore - public long getCreatedTimeMillis() { - return getGivenDateField(CREATED); - } - - @JsonIgnore - public long getUpdatedTimeMillis() { - return getGivenDateField(UPDATED); - } - - @JsonIgnore - private long getGivenDateField(String dateTimeFieldToPull) { - long dateTimeField = 0; - - if (fields != null && Objects.nonNull(fields.get(dateTimeFieldToPull)) && JiraDateTimePattern.matcher(fields.get(dateTimeFieldToPull) - .toString()).matches()) { - String charSequence = fields.get(dateTimeFieldToPull).toString(); - OffsetDateTime offsetDateTime = OffsetDateTime.parse(charSequence, offsetDateTimeFormatter); - new Date(offsetDateTime.toInstant().toEpochMilli()); - dateTimeField = offsetDateTime.toEpochSecond() * 1000; - } - return dateTimeField; - } - - -} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/SpaceItem.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/SpaceItem.java new file mode 100644 index 0000000000..9e8733b582 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/SpaceItem.java @@ -0,0 +1,61 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.models; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Getter; +import lombok.Setter; + + +@Setter +@Getter +@JsonIgnoreProperties(ignoreUnknown = true) +public class SpaceItem { + + /** + * The ID of the issue. + */ + @JsonProperty("id") + private int id; + + /** + * The type of the issue. + */ + @JsonProperty("key") + private String key = null; + + /** + * The type of the issue. + */ + @JsonProperty("alias") + private String alias = null; + + /** + * The type of the issue. + */ + @JsonProperty("name") + private String name = null; + + /** + * The type of the issue. + */ + @JsonProperty("status") + private String status = null; + + /** + * The type of the issue. + */ + @JsonProperty("type") + private String type = null; + + +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClient.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClient.java index 9152b88a06..f0bf79be14 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClient.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClient.java @@ -16,10 +16,9 @@ import io.micrometer.core.instrument.Timer; import lombok.extern.slf4j.Slf4j; import org.opensearch.dataprepper.metrics.PluginMetrics; -import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceSourceConfig; import org.opensearch.dataprepper.plugins.source.confluence.exception.BadRequestException; import org.opensearch.dataprepper.plugins.source.confluence.exception.UnAuthorizedException; -import org.opensearch.dataprepper.plugins.source.confluence.models.SearchResults; +import org.opensearch.dataprepper.plugins.source.confluence.models.ConfluenceSearchResults; import org.opensearch.dataprepper.plugins.source.confluence.rest.auth.ConfluenceAuthConfig; import org.opensearch.dataprepper.plugins.source.confluence.utils.AddressValidation; import org.springframework.http.HttpStatus; @@ -34,78 +33,76 @@ import static org.opensearch.dataprepper.logging.DataPrepperMarkers.NOISY; import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.RETRY_ATTEMPT; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.EXPAND_FIELD; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.EXPAND_VALUE; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.JQL_FIELD; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.CqlConstants.CQL_FIELD; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.CqlConstants.EXPAND_FIELD; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.CqlConstants.EXPAND_VALUE; @Slf4j @Named public class ConfluenceRestClient { - public static final String REST_API_SEARCH = "rest/api/3/search"; - public static final String REST_API_FETCH_ISSUE = "rest/api/3/issue"; - public static final String REST_API_PROJECTS = "/rest/api/3/project/search"; + public static final String REST_API_SEARCH = "wiki/rest/api/content/search"; + public static final String REST_API_FETCH_CONTENT = "wiki/rest/api/content/"; + public static final String REST_API_CONTENT_EXPAND_PARAM = "?expand=body.view"; + //public static final String REST_API_SPACES = "/rest/api/api/spaces"; public static final String FIFTY = "50"; public static final String START_AT = "startAt"; public static final String MAX_RESULT = "maxResults"; public static final List RETRY_ATTEMPT_SLEEP_TIME = List.of(1, 2, 5, 10, 20, 40); - private static final String TICKET_FETCH_LATENCY_TIMER = "ticketFetchLatency"; + private static final String PAGE_FETCH_LATENCY_TIMER = "pageFetchLatency"; private static final String SEARCH_CALL_LATENCY_TIMER = "searchCallLatency"; - private static final String PROJECTS_FETCH_LATENCY_TIMER = "projectFetchLatency"; - private static final String ISSUES_REQUESTED = "issuesRequested"; + private static final String SPACES_FETCH_LATENCY_TIMER = "spacesFetchLatency"; + private static final String PAGES_REQUESTED = "pagesRequested"; private int sleepTimeMultiplier = 1000; private final RestTemplate restTemplate; private final ConfluenceAuthConfig authConfig; - private final Timer ticketFetchLatencyTimer; + private final Timer contentFetchLatencyTimer; private final Timer searchCallLatencyTimer; - private final Timer projectFetchLatencyTimer; - private final Counter issuesRequestedCounter; - private final PluginMetrics jiraPluginMetrics = PluginMetrics.fromNames("jiraRestClient", "aws"); + private final Timer spaceFetchLatencyTimer; + private final Counter contentRequestedCounter; - public ConfluenceRestClient(RestTemplate restTemplate, ConfluenceAuthConfig authConfig) { + public ConfluenceRestClient(RestTemplate restTemplate, ConfluenceAuthConfig authConfig, + PluginMetrics pluginMetrics) { this.restTemplate = restTemplate; this.authConfig = authConfig; - ticketFetchLatencyTimer = jiraPluginMetrics.timer(TICKET_FETCH_LATENCY_TIMER); - searchCallLatencyTimer = jiraPluginMetrics.timer(SEARCH_CALL_LATENCY_TIMER); - projectFetchLatencyTimer = jiraPluginMetrics.timer(PROJECTS_FETCH_LATENCY_TIMER); - - issuesRequestedCounter = jiraPluginMetrics.counter(ISSUES_REQUESTED); + contentFetchLatencyTimer = pluginMetrics.timer(PAGE_FETCH_LATENCY_TIMER); + searchCallLatencyTimer = pluginMetrics.timer(SEARCH_CALL_LATENCY_TIMER); + spaceFetchLatencyTimer = pluginMetrics.timer(SPACES_FETCH_LATENCY_TIMER); + contentRequestedCounter = pluginMetrics.counter(PAGES_REQUESTED); } /** * Method to get Issues. * - * @param jql input parameter. - * @param startAt the start at - * @param configuration input parameter. + * @param jql input parameter. + * @param startAt the start at * @return InputStream input stream */ @Timed(SEARCH_CALL_LATENCY_TIMER) - public SearchResults getAllIssues(StringBuilder jql, int startAt, - ConfluenceSourceConfig configuration) { + public ConfluenceSearchResults getAllContent(StringBuilder jql, int startAt) { String url = authConfig.getUrl() + REST_API_SEARCH; URI uri = UriComponentsBuilder.fromHttpUrl(url) .queryParam(MAX_RESULT, FIFTY) .queryParam(START_AT, startAt) - .queryParam(JQL_FIELD, jql) + .queryParam(CQL_FIELD, jql) .queryParam(EXPAND_FIELD, EXPAND_VALUE) .buildAndExpand().toUri(); - return invokeRestApi(uri, SearchResults.class).getBody(); + return invokeRestApi(uri, ConfluenceSearchResults.class).getBody(); } /** * Gets issue. * - * @param issueKey the item info + * @param contentId the item info * @return the issue */ - @Timed(TICKET_FETCH_LATENCY_TIMER) - public String getIssue(String issueKey) { - issuesRequestedCounter.increment(); - String url = authConfig.getUrl() + REST_API_FETCH_ISSUE + "/" + issueKey; + @Timed(PAGE_FETCH_LATENCY_TIMER) + public String getContent(String contentId) { + contentRequestedCounter.increment(); + String url = authConfig.getUrl() + REST_API_FETCH_CONTENT + "/" + contentId + REST_API_CONTENT_EXPAND_PARAM; URI uri = UriComponentsBuilder.fromHttpUrl(url).buildAndExpand().toUri(); return invokeRestApi(uri, String.class).getBody(); } diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceOauthConfig.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceOauthConfig.java index 51638b4dcb..dea42f2917 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceOauthConfig.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceOauthConfig.java @@ -31,7 +31,7 @@ import java.util.Map; import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.RETRY_ATTEMPT; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.SLASH; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.CqlConstants.SLASH; /** * The type Jira service. diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/ConfluenceConfigHelper.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/ConfluenceConfigHelper.java index 09e334eea0..9f36bfe91d 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/ConfluenceConfigHelper.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/ConfluenceConfigHelper.java @@ -21,24 +21,24 @@ import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.OAUTH2; /** - * The type Jira configuration. + * The type Confluence configuration. */ @Slf4j public class ConfluenceConfigHelper { /** - * Get Issue Types Filter from repository configuration. + * Get Content Types Filter from configuration. * - * @return List Issue Type Filter. + * @return List Content Type Filter. */ - public static List getIssueTypeIncludeFilter(ConfluenceSourceConfig repositoryConfiguration) { + public static List getContentTypeIncludeFilter(ConfluenceSourceConfig repositoryConfiguration) { if (repositoryConfiguration.getFilterConfig() == null || repositoryConfiguration.getFilterConfig().getPageTypeConfig() == null) { return new ArrayList<>(); } return repositoryConfiguration.getFilterConfig().getPageTypeConfig().getInclude(); } - public static List getIssueTypeExcludeFilter(ConfluenceSourceConfig repositoryConfiguration) { + public static List getContentTypeExcludeFilter(ConfluenceSourceConfig repositoryConfiguration) { if (repositoryConfiguration.getFilterConfig() == null || repositoryConfiguration.getFilterConfig().getPageTypeConfig() == null) { return new ArrayList<>(); } @@ -46,12 +46,11 @@ public static List getIssueTypeExcludeFilter(ConfluenceSourceConfig repo } /** - * Get Project Filter Types from repository configuration. - * public static final String ST = "status"; + * Get Space Filter Types from configuration. * - * @return List Project Filter. + * @return List Space Filter. */ - public static List getProjectNameIncludeFilter(ConfluenceSourceConfig repositoryConfiguration) { + public static List getSpacesNameIncludeFilter(ConfluenceSourceConfig repositoryConfiguration) { if (repositoryConfiguration.getFilterConfig() == null || repositoryConfiguration.getFilterConfig().getSpaceConfig() == null || repositoryConfiguration.getFilterConfig().getSpaceConfig().getNameConfig() == null) { @@ -60,7 +59,7 @@ public static List getProjectNameIncludeFilter(ConfluenceSourceConfig re return repositoryConfiguration.getFilterConfig().getSpaceConfig().getNameConfig().getInclude(); } - public static List getProjectNameExcludeFilter(ConfluenceSourceConfig repositoryConfiguration) { + public static List getSpacesNameExcludeFilter(ConfluenceSourceConfig repositoryConfiguration) { if (repositoryConfiguration.getFilterConfig() == null || repositoryConfiguration.getFilterConfig().getSpaceConfig() == null || repositoryConfiguration.getFilterConfig().getSpaceConfig().getNameConfig() == null) { @@ -85,7 +84,7 @@ public static boolean validateConfig(ConfluenceSourceConfig config) { if (BASIC.equals(authType)) { if (config.getAuthenticationConfig().getBasicConfig().getUsername() == null || config.getAuthenticationConfig().getBasicConfig().getPassword() == null) { - throw new RuntimeException("Jira ID or Credential are required for Basic AuthType"); + throw new RuntimeException("Confluence ID or Credential are required for Basic AuthType"); } } diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/Constants.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/Constants.java index dafa4c3f90..114c33cebd 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/Constants.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/Constants.java @@ -19,14 +19,12 @@ public class Constants { public static final String KEY = "key"; public static final String NAME = "name"; - public static final String PROJECT = "project"; + public static final String SPACE = "space"; public static final String OAUTH2 = "OAuth2"; - public static final String _PROJECT = "project-"; - public static final String _ISSUE = "ISSUE-"; - public static final String UPDATED = "updated"; - public static final String PROJECT_KEY = "j_project_key"; - public static final String PROJECT_NAME = "j_project_name"; - public static final String ISSUE_KEY = "j_issue_key"; + public static final String LAST_MODIFIED = "lastModified"; + public static final String SPACE_KEY = "space"; + public static final String CONTENT_TITLE = "title"; + public static final String CONTENT_ID = "id"; public static final String CREATED = "created"; public static final String BASIC = "Basic"; public static final String LIVE = "live"; diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/JqlConstants.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/CqlConstants.java similarity index 55% rename from data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/JqlConstants.java rename to data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/CqlConstants.java index 7dfe523df2..74b3a0f6c7 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/JqlConstants.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/CqlConstants.java @@ -10,21 +10,19 @@ package org.opensearch.dataprepper.plugins.source.confluence.utils; -public class JqlConstants { +public class CqlConstants { public static final String GREATER_THAN_EQUALS = ">="; public static final String CLOSING_ROUND_BRACKET = ")"; public static final String SLASH = "/"; - public static final String PROJECT_IN = " AND project in ("; - public static final String STATUS_IN = " AND status in ("; - public static final String PROJECT_NOT_IN = " AND project not in ("; - public static final String STATUS_NOT_IN = " AND status not in ("; + public static final String SPACE_IN = " AND space in ("; + public static final String SPACE_NOT_IN = " AND space not in ("; public static final String DELIMITER = "\",\""; public static final String PREFIX = "\""; public static final String SUFFIX = "\""; - public static final String ISSUE_TYPE_IN = " AND issueType in ("; - public static final String ISSUE_TYPE_NOT_IN = " AND issueType not in ("; - public static final String JQL_FIELD = "jql"; + public static final String CONTENT_TYPE_IN = " AND type in ("; + public static final String CONTENT_TYPE_NOT_IN = " AND type not in ("; + public static final String CQL_FIELD = "cql"; public static final String EXPAND_FIELD = "expand"; - public static final String EXPAND_VALUE = "all"; + public static final String EXPAND_VALUE = "all,space"; } diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceClientTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceClientTest.java index a75e6dfbc2..efd8e20303 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceClientTest.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceClientTest.java @@ -22,10 +22,6 @@ import org.opensearch.dataprepper.model.buffer.Buffer; import org.opensearch.dataprepper.model.event.Event; import org.opensearch.dataprepper.model.record.Record; -import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceClient; -import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceIterator; -import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceService; -import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceSourceConfig; import org.opensearch.dataprepper.plugins.source.source_crawler.base.PluginExecutorServiceProvider; import org.opensearch.dataprepper.plugins.source.source_crawler.coordination.state.SaasWorkerProgressState; @@ -92,7 +88,7 @@ void testExecutePartition() throws Exception { Instant exportStartTime = Instant.now(); when(saasWorkerProgressState.getExportStartTime()).thenReturn(Instant.ofEpochSecond(exportStartTime.toEpochMilli())); - when(confluenceService.getIssue(anyString())).thenReturn("{\"id\":\"ID1\",\"key\":\"TEST-1\"}"); + when(confluenceService.getContent(anyString())).thenReturn("{\"id\":\"ID1\",\"key\":\"TEST-1\"}"); ArgumentCaptor>> recordsCaptor = ArgumentCaptor.forClass((Class) Collection.class); @@ -117,7 +113,7 @@ void testExecutePartitionError() throws Exception { Instant exportStartTime = Instant.now(); when(saasWorkerProgressState.getExportStartTime()).thenReturn(Instant.ofEpochSecond(exportStartTime.toEpochMilli())); - when(confluenceService.getIssue(anyString())).thenReturn("{\"id\":\"ID1\",\"key\":\"TEST-1\"}"); + when(confluenceService.getContent(anyString())).thenReturn("{\"id\":\"ID1\",\"key\":\"TEST-1\"}"); ObjectMapper mockObjectMapper = mock(ObjectMapper.class); @@ -139,7 +135,7 @@ void bufferWriteRuntimeTest() throws Exception { Instant exportStartTime = Instant.now(); when(saasWorkerProgressState.getExportStartTime()).thenReturn(Instant.ofEpochSecond(exportStartTime.toEpochMilli())); - when(confluenceService.getIssue(anyString())).thenReturn("{\"id\":\"ID1\",\"key\":\"TEST-1\"}"); + when(confluenceService.getContent(anyString())).thenReturn("{\"id\":\"ID1\",\"key\":\"TEST-1\"}"); ArgumentCaptor>> recordsCaptor = ArgumentCaptor.forClass((Class) Collection.class); diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceConfigHelperTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceConfigHelperTest.java index d83967cf0b..57ea307b5c 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceConfigHelperTest.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceConfigHelperTest.java @@ -44,9 +44,6 @@ public class ConfluenceConfigHelperTest { @Mock FilterConfig filterConfig; - @Mock - StatusConfig statusConfig; - @Mock PageTypeConfig pageTypeConfig; @@ -76,20 +73,20 @@ void testInitialization() { ConfluenceConfigHelper confluenceConfigHelper = new ConfluenceConfigHelper(); assertNotNull(confluenceConfigHelper); } - + @Test void testGetIssueTypeFilter() { when(confluenceSourceConfig.getFilterConfig()).thenReturn(filterConfig); when(filterConfig.getPageTypeConfig()).thenReturn(pageTypeConfig); - assertTrue(ConfluenceConfigHelper.getIssueTypeIncludeFilter(confluenceSourceConfig).isEmpty()); - assertTrue(ConfluenceConfigHelper.getIssueTypeExcludeFilter(confluenceSourceConfig).isEmpty()); + assertTrue(ConfluenceConfigHelper.getContentTypeIncludeFilter(confluenceSourceConfig).isEmpty()); + assertTrue(ConfluenceConfigHelper.getContentTypeExcludeFilter(confluenceSourceConfig).isEmpty()); List issueTypeFilter = List.of("Bug", "Story"); List issueTypeExcludeFilter = List.of("Bug2", "Story2"); when(pageTypeConfig.getInclude()).thenReturn(issueTypeFilter); when(pageTypeConfig.getExclude()).thenReturn(issueTypeExcludeFilter); - assertEquals(issueTypeFilter, ConfluenceConfigHelper.getIssueTypeIncludeFilter(confluenceSourceConfig)); - assertEquals(issueTypeExcludeFilter, ConfluenceConfigHelper.getIssueTypeExcludeFilter(confluenceSourceConfig)); + assertEquals(issueTypeFilter, ConfluenceConfigHelper.getContentTypeIncludeFilter(confluenceSourceConfig)); + assertEquals(issueTypeExcludeFilter, ConfluenceConfigHelper.getContentTypeExcludeFilter(confluenceSourceConfig)); } @Test @@ -97,14 +94,14 @@ void testGetProjectNameFilter() { when(confluenceSourceConfig.getFilterConfig()).thenReturn(filterConfig); when(filterConfig.getSpaceConfig()).thenReturn(spaceConfig); when(spaceConfig.getNameConfig()).thenReturn(nameConfig); - assertTrue(ConfluenceConfigHelper.getProjectNameIncludeFilter(confluenceSourceConfig).isEmpty()); - assertTrue(ConfluenceConfigHelper.getProjectNameExcludeFilter(confluenceSourceConfig).isEmpty()); + assertTrue(ConfluenceConfigHelper.getSpacesNameIncludeFilter(confluenceSourceConfig).isEmpty()); + assertTrue(ConfluenceConfigHelper.getSpacesNameExcludeFilter(confluenceSourceConfig).isEmpty()); List projectNameFilter = List.of("TEST", "TEST2"); List projectNameExcludeFilter = List.of("TEST3", "TEST4"); when(nameConfig.getInclude()).thenReturn(projectNameFilter); when(nameConfig.getExclude()).thenReturn(projectNameExcludeFilter); - assertEquals(projectNameFilter, ConfluenceConfigHelper.getProjectNameIncludeFilter(confluenceSourceConfig)); - assertEquals(projectNameExcludeFilter, ConfluenceConfigHelper.getProjectNameExcludeFilter(confluenceSourceConfig)); + assertEquals(projectNameFilter, ConfluenceConfigHelper.getSpacesNameIncludeFilter(confluenceSourceConfig)); + assertEquals(projectNameExcludeFilter, ConfluenceConfigHelper.getSpacesNameExcludeFilter(confluenceSourceConfig)); } diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceItemInfoTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceItemInfoTest.java index f10249b686..26ef017dec 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceItemInfoTest.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceItemInfoTest.java @@ -15,7 +15,6 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceItemInfo; import org.opensearch.dataprepper.plugins.source.confluence.utils.Constants; import java.time.Instant; @@ -59,8 +58,8 @@ void setUP() { void testGetters() { assertEquals(confluenceItemInfo.getItemId(), itemId); assertEquals(confluenceItemInfo.getId(), id); - assertEquals(confluenceItemInfo.getProject(), project); - assertEquals(confluenceItemInfo.getIssueType(), issueType); + assertEquals(confluenceItemInfo.getSpace(), project); + assertEquals(confluenceItemInfo.getContentType(), issueType); assertEquals(confluenceItemInfo.getMetadata(), metadata); assertEquals(confluenceItemInfo.getEventTime(), eventTime); } @@ -78,12 +77,12 @@ void testSetter() { assertNotEquals(confluenceItemInfo.getItemId(), itemId); confluenceItemInfo.setId("newID"); assertNotEquals(confluenceItemInfo.getId(), id); - confluenceItemInfo.setProject("newProject"); - assertNotEquals(confluenceItemInfo.getProject(), project); + confluenceItemInfo.setSpace("newProject"); + assertNotEquals(confluenceItemInfo.getSpace(), project); confluenceItemInfo.setMetadata(newMetadata); assertNotEquals(confluenceItemInfo.getMetadata(), metadata); - confluenceItemInfo.setIssueType("newIssueType"); - assertNotEquals(confluenceItemInfo.getIssueType(), issueType); + confluenceItemInfo.setContentType("newIssueType"); + assertNotEquals(confluenceItemInfo.getContentType(), issueType); } @@ -97,11 +96,11 @@ void testGetPartitionKey() { @Test void testGetLastModifiedAt() { - when(metadata.get(Constants.UPDATED)).thenReturn("5"); + when(metadata.get(Constants.LAST_MODIFIED)).thenReturn("5"); when(metadata.get(Constants.CREATED)).thenReturn("0"); assertEquals(Instant.ofEpochMilli(5), confluenceItemInfo.getLastModifiedAt()); - when(metadata.get(Constants.UPDATED)).thenReturn("5"); + when(metadata.get(Constants.LAST_MODIFIED)).thenReturn("5"); when(metadata.get(Constants.CREATED)).thenReturn("7"); assertEquals(Instant.ofEpochMilli(7), confluenceItemInfo.getLastModifiedAt()); } diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceIteratorTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceIteratorTest.java index ac64972c72..cc2bfde044 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceIteratorTest.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceIteratorTest.java @@ -15,38 +15,26 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.dataprepper.plugins.source.confluence.models.IssueBean; -import org.opensearch.dataprepper.plugins.source.confluence.models.SearchResults; +import org.opensearch.dataprepper.plugins.source.confluence.models.ConfluenceSearchResults; import org.opensearch.dataprepper.plugins.source.confluence.rest.ConfluenceRestClient; import org.opensearch.dataprepper.plugins.source.source_crawler.base.PluginExecutorServiceProvider; import java.time.Instant; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.UUID; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.when; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.CREATED; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.KEY; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.UPDATED; @ExtendWith(MockitoExtension.class) public class ConfluenceIteratorTest { @Mock - private SearchResults mockSearchResults; + private ConfluenceSearchResults mockConfluenceSearchResults; @Mock private ConfluenceRestClient confluenceRestClient; private ConfluenceService confluenceService; @@ -69,9 +57,7 @@ void testInitialization() { confluenceIterator = createObjectUnderTest(); assertNotNull(confluenceIterator); confluenceIterator.initialize(Instant.ofEpochSecond(0)); - when(mockSearchResults.getIssues()).thenReturn(new ArrayList<>()); - when(mockSearchResults.getTotal()).thenReturn(0); - doReturn(mockSearchResults).when(confluenceRestClient).getAllIssues(any(StringBuilder.class), anyInt(), any(ConfluenceSourceConfig.class)); + doReturn(mockConfluenceSearchResults).when(confluenceRestClient).getAllContent(any(StringBuilder.class), anyInt()); assertFalse(confluenceIterator.hasNext()); } @@ -94,21 +80,6 @@ void sleepInterruptionTest() { testThread.interrupt(); } - @Test - void testItemInfoQueueNotEmpty() { - confluenceIterator = createObjectUnderTest(); - List mockIssues = new ArrayList<>(); - IssueBean issue1 = createIssueBean(false); - mockIssues.add(issue1); - when(mockSearchResults.getIssues()).thenReturn(mockIssues); - when(mockSearchResults.getTotal()).thenReturn(0); - doReturn(mockSearchResults).when(confluenceRestClient).getAllIssues(any(StringBuilder.class), anyInt(), any(ConfluenceSourceConfig.class)); - - confluenceIterator.initialize(Instant.ofEpochSecond(0)); - confluenceIterator.setCrawlerQWaitTimeMillis(1); - assertTrue(confluenceIterator.hasNext()); - assertNotNull(confluenceIterator.next()); - } @Test void testStartCrawlerThreads() { @@ -119,83 +90,5 @@ void testStartCrawlerThreads() { assertEquals(1, confluenceIterator.showFutureList().size()); } - @Test - void testFuturesCompleted() throws InterruptedException { - confluenceIterator = createObjectUnderTest(); - List mockIssues = new ArrayList<>(); - IssueBean issue1 = createIssueBean(false); - mockIssues.add(issue1); - IssueBean issue2 = createIssueBean(false); - mockIssues.add(issue2); - IssueBean issue3 = createIssueBean(false); - mockIssues.add(issue3); - when(mockSearchResults.getIssues()).thenReturn(mockIssues); - when(mockSearchResults.getTotal()).thenReturn(0); - doReturn(mockSearchResults).when(confluenceRestClient).getAllIssues(any(StringBuilder.class), anyInt(), any(ConfluenceSourceConfig.class)); - confluenceIterator.initialize(Instant.ofEpochSecond(0)); - confluenceIterator.setCrawlerQWaitTimeMillis(1); - confluenceIterator.hasNext(); - - Thread.sleep(1); - confluenceIterator.showFutureList().forEach(future -> assertTrue(future.isDone())); - assertEquals(confluenceIterator.showItemInfoQueue().size(), mockIssues.size()); - } - - @Test - void testItemInfoQueueEmpty() { - confluenceIterator = createObjectUnderTest(); - List mockIssues = new ArrayList<>(); - when(mockSearchResults.getIssues()).thenReturn(mockIssues); - when(mockSearchResults.getTotal()).thenReturn(0); - doReturn(mockSearchResults).when(confluenceRestClient).getAllIssues(any(StringBuilder.class), anyInt(), any(ConfluenceSourceConfig.class)); - - confluenceIterator.initialize(Instant.ofEpochSecond(0)); - confluenceIterator.setCrawlerQWaitTimeMillis(1); - assertFalse(confluenceIterator.hasNext()); - assertThrows(NoSuchElementException.class, () -> confluenceIterator.next()); - } - - - private IssueBean createIssueBean(boolean nullFields) { - IssueBean issue1 = new IssueBean(); - issue1.setId(UUID.randomUUID().toString()); - issue1.setKey("issue_1_key"); - issue1.setSelf("https://example.com/rest/api/2/issue/123"); - issue1.setExpand("operations,versionedRepresentations,editmeta"); - - Map fieldMap = new HashMap<>(); - if (!nullFields) { - fieldMap.put(CREATED, Instant.now()); - fieldMap.put(UPDATED, Instant.now()); - } else { - fieldMap.put(CREATED, 0); - fieldMap.put(UPDATED, 0); - } - - Map issueTypeMap = new HashMap<>(); - issueTypeMap.put("name", "Task"); - issueTypeMap.put("self", "https://example.com/rest/api/2/issuetype/1"); - issueTypeMap.put("id", "1"); - fieldMap.put("issuetype", issueTypeMap); - - Map projectMap = new HashMap<>(); - if (!nullFields) { - projectMap.put("name", "project name test"); - projectMap.put(KEY, "TEST"); - } - fieldMap.put("project", projectMap); - - Map priorityMap = new HashMap<>(); - priorityMap.put("name", "Medium"); - fieldMap.put("priority", priorityMap); - - Map statusMap = new HashMap<>(); - statusMap.put("name", "In Progress"); - fieldMap.put("status", statusMap); - - issue1.setFields(fieldMap); - - return issue1; - } } diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceServiceTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceServiceTest.java index 9859ea3051..df4b9d32be 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceServiceTest.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceServiceTest.java @@ -21,8 +21,8 @@ import org.opensearch.dataprepper.model.plugin.PluginConfigVariable; import org.opensearch.dataprepper.plugins.source.confluence.configuration.Oauth2Config; import org.opensearch.dataprepper.plugins.source.confluence.exception.BadRequestException; -import org.opensearch.dataprepper.plugins.source.confluence.models.IssueBean; -import org.opensearch.dataprepper.plugins.source.confluence.models.SearchResults; +import org.opensearch.dataprepper.plugins.source.confluence.models.ConfluenceItem; +import org.opensearch.dataprepper.plugins.source.confluence.models.ConfluenceSearchResults; import org.opensearch.dataprepper.plugins.source.confluence.rest.ConfluenceRestClient; import org.opensearch.dataprepper.plugins.source.confluence.utils.MockPluginConfigVariableImpl; import org.opensearch.dataprepper.plugins.source.source_crawler.base.PluginExecutorServiceProvider; @@ -58,10 +58,10 @@ import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.BASIC; import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.CREATED; import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.KEY; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.LAST_MODIFIED; import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.NAME; import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.OAUTH2; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.PROJECT; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.UPDATED; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.SPACE; /** @@ -178,12 +178,12 @@ void testJiraServiceInitialization() throws JsonProcessingException { ConfluenceSourceConfig confluenceSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey); ConfluenceService confluenceService = new ConfluenceService(confluenceSourceConfig, confluenceRestClient); assertNotNull(confluenceService); - when(confluenceRestClient.getIssue(anyString())).thenReturn("test String"); - assertNotNull(confluenceService.getIssue("test Key")); + when(confluenceRestClient.getContent(anyString())).thenReturn("test String"); + assertNotNull(confluenceService.getContent("test Key")); } @Test - public void testGetJiraEntities() throws JsonProcessingException { + public void testGetPages() throws JsonProcessingException { List issueType = new ArrayList<>(); List issueStatus = new ArrayList<>(); List projectKey = new ArrayList<>(); @@ -192,23 +192,21 @@ public void testGetJiraEntities() throws JsonProcessingException { projectKey.add("KAN"); ConfluenceSourceConfig confluenceSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey); ConfluenceService confluenceService = spy(new ConfluenceService(confluenceSourceConfig, confluenceRestClient)); - List mockIssues = new ArrayList<>(); - IssueBean issue1 = createIssueBean(false, false); + List mockIssues = new ArrayList<>(); + ConfluenceItem issue1 = createConfluenceItemBean(false, false); mockIssues.add(issue1); - IssueBean issue2 = createIssueBean(true, false); + ConfluenceItem issue2 = createConfluenceItemBean(true, false); mockIssues.add(issue2); - IssueBean issue3 = createIssueBean(false, true); + ConfluenceItem issue3 = createConfluenceItemBean(false, true); mockIssues.add(issue3); - SearchResults mockSearchResults = mock(SearchResults.class); - when(mockSearchResults.getIssues()).thenReturn(mockIssues); - when(mockSearchResults.getTotal()).thenReturn(mockIssues.size()); + ConfluenceSearchResults mockConfluenceSearchResults = mock(ConfluenceSearchResults.class); - doReturn(mockSearchResults).when(confluenceRestClient).getAllIssues(any(StringBuilder.class), anyInt(), any(ConfluenceSourceConfig.class)); + doReturn(mockConfluenceSearchResults).when(confluenceRestClient).getAllContent(any(StringBuilder.class), anyInt()); Instant timestamp = Instant.ofEpochSecond(0); Queue itemInfoQueue = new ConcurrentLinkedQueue<>(); - confluenceService.getJiraEntities(confluenceSourceConfig, timestamp, itemInfoQueue); + confluenceService.getPages(confluenceSourceConfig, timestamp, itemInfoQueue); assertEquals(mockIssues.size(), itemInfoQueue.size()); } @@ -220,21 +218,20 @@ public void buildIssueItemInfoMultipleFutureThreads() throws JsonProcessingExcep issueType.add("Task"); ConfluenceSourceConfig confluenceSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey); ConfluenceService confluenceService = spy(new ConfluenceService(confluenceSourceConfig, confluenceRestClient)); - List mockIssues = new ArrayList<>(); + List mockIssues = new ArrayList<>(); for (int i = 0; i < 50; i++) { - IssueBean issue1 = createIssueBean(false, false); + ConfluenceItem issue1 = createConfluenceItemBean(false, false); mockIssues.add(issue1); } - SearchResults mockSearchResults = mock(SearchResults.class); - when(mockSearchResults.getIssues()).thenReturn(mockIssues); - when(mockSearchResults.getTotal()).thenReturn(100); + ConfluenceSearchResults mockConfluenceSearchResults = mock(ConfluenceSearchResults.class); - doReturn(mockSearchResults).when(confluenceRestClient).getAllIssues(any(StringBuilder.class), anyInt(), any(ConfluenceSourceConfig.class)); + + doReturn(mockConfluenceSearchResults).when(confluenceRestClient).getAllContent(any(StringBuilder.class), anyInt()); Instant timestamp = Instant.ofEpochSecond(0); Queue itemInfoQueue = new ConcurrentLinkedQueue<>(); - confluenceService.getJiraEntities(confluenceSourceConfig, timestamp, itemInfoQueue); + confluenceService.getPages(confluenceSourceConfig, timestamp, itemInfoQueue); assertTrue(itemInfoQueue.size() >= 100); } @@ -256,11 +253,11 @@ public void testBadProjectKeys() throws JsonProcessingException { Instant timestamp = Instant.ofEpochSecond(0); Queue itemInfoQueue = new ConcurrentLinkedQueue<>(); - assertThrows(BadRequestException.class, () -> confluenceService.getJiraEntities(confluenceSourceConfig, timestamp, itemInfoQueue)); + assertThrows(BadRequestException.class, () -> confluenceService.getPages(confluenceSourceConfig, timestamp, itemInfoQueue)); } @Test - public void testGetJiraEntitiesException() throws JsonProcessingException { + public void testGetPagesException() throws JsonProcessingException { List issueType = new ArrayList<>(); List issueStatus = new ArrayList<>(); List projectKey = new ArrayList<>(); @@ -268,29 +265,27 @@ public void testGetJiraEntitiesException() throws JsonProcessingException { ConfluenceSourceConfig confluenceSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey); ConfluenceService confluenceService = spy(new ConfluenceService(confluenceSourceConfig, confluenceRestClient)); - doThrow(RuntimeException.class).when(confluenceRestClient).getAllIssues(any(StringBuilder.class), anyInt(), any(ConfluenceSourceConfig.class)); + doThrow(RuntimeException.class).when(confluenceRestClient).getAllContent(any(StringBuilder.class), anyInt()); Instant timestamp = Instant.ofEpochSecond(0); Queue itemInfoQueue = new ConcurrentLinkedQueue<>(); - assertThrows(RuntimeException.class, () -> confluenceService.getJiraEntities(confluenceSourceConfig, timestamp, itemInfoQueue)); + assertThrows(RuntimeException.class, () -> confluenceService.getPages(confluenceSourceConfig, timestamp, itemInfoQueue)); } - private IssueBean createIssueBean(boolean nullFields, boolean createdNull) { - IssueBean issue1 = new IssueBean(); + private ConfluenceItem createConfluenceItemBean(boolean nullFields, boolean createdNull) { + ConfluenceItem issue1 = new ConfluenceItem(); issue1.setId(UUID.randomUUID().toString()); - issue1.setKey("issue_1_key"); - issue1.setSelf("https://example.com/rest/api/2/issue/123"); - issue1.setExpand("operations,versionedRepresentations,editmeta"); + issue1.setTitle("issue_1_key"); Map fieldMap = new HashMap<>(); if (!nullFields) { fieldMap.put(CREATED, "2024-07-06T21:12:23.437-0700"); - fieldMap.put(UPDATED, "2024-07-06T21:12:23.106-0700"); + fieldMap.put(LAST_MODIFIED, "2024-07-06T21:12:23.106-0700"); } else { fieldMap.put(CREATED, 0); - fieldMap.put(UPDATED, 0); + fieldMap.put(LAST_MODIFIED, 0); } if (createdNull) { fieldMap.put(CREATED, null); @@ -307,7 +302,7 @@ private IssueBean createIssueBean(boolean nullFields, boolean createdNull) { projectMap.put(NAME, "project name test"); projectMap.put(KEY, "TEST"); } - fieldMap.put(PROJECT, projectMap); + fieldMap.put(SPACE, projectMap); Map priorityMap = new HashMap<>(); priorityMap.put(NAME, "Medium"); @@ -317,8 +312,6 @@ private IssueBean createIssueBean(boolean nullFields, boolean createdNull) { statusMap.put(NAME, "In Progress"); fieldMap.put("statuses", statusMap); - issue1.setFields(fieldMap); - return issue1; } diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceConfigTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceConfigTest.java index ddf84b8004..73fdbe9dbe 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceConfigTest.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceConfigTest.java @@ -114,7 +114,6 @@ void testGetters() throws Exception { confluenceSourceConfig = createJiraSourceConfig(BASIC, false); assertEquals(confluenceSourceConfig.getFilterConfig().getPageTypeConfig().getInclude(), issueTypeList); assertEquals(confluenceSourceConfig.getFilterConfig().getSpaceConfig().getNameConfig().getInclude(), projectList); - assertEquals(confluenceSourceConfig.getFilterConfig().getStatusConfig().getInclude(), statusList); assertEquals(confluenceSourceConfig.getAccountUrl(), accountUrl); assertEquals(confluenceSourceConfig.getAuthenticationConfig().getBasicConfig().getPassword(), password); assertEquals(confluenceSourceConfig.getAuthenticationConfig().getBasicConfig().getUsername(), username); diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceItemTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceItemTest.java new file mode 100644 index 0000000000..1c22fb673a --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceItemTest.java @@ -0,0 +1,71 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.models; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +@ExtendWith(MockitoExtension.class) +public class ConfluenceItemTest { + + + private ConfluenceItem issueBean; + + @BeforeEach + void setup() { + issueBean = new ConfluenceItem(); + } + + @Test + public void testInitialization() { + assertNotNull(issueBean); + } + + @Test + public void testNull() { + assertNull(issueBean.getId()); + } + + @Test + void testNullCases() { + assertEquals(issueBean.getUpdatedTimeMillis(), 0); + } + + @Test + void testGivenDateField() { + Map fieldsTestObject = new HashMap<>(); + fieldsTestObject.put("created", "2024-07-06T21:12:23.437-0700"); + fieldsTestObject.put("updated", "2022-07-06T21:12:23.106-0700"); + assertEquals(issueBean.getCreatedTimeMillis(), 1720325543000L); + assertEquals(issueBean.getUpdatedTimeMillis(), 1657167143000L); + } + + @Test + public void testStringSettersAndGetters() { + String self = "selfTest"; + String key = "keyTest"; + String id = "idTest"; + String expand = "expandTest"; + + issueBean.setId(id); + assertEquals(issueBean.getId(), id); + } + +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceSearchResultsTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceSearchResultsTest.java new file mode 100644 index 0000000000..fe070c8b98 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceSearchResultsTest.java @@ -0,0 +1,41 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.models; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; + +import static org.junit.jupiter.api.Assertions.assertNotNull; + +@ExtendWith(MockitoExtension.class) +public class ConfluenceSearchResultsTest { + + private ConfluenceSearchResults confluenceSearchResults; + private final ObjectMapper objectMapper = new ObjectMapper(); + + @BeforeEach + public void setUp() throws JsonProcessingException { + String state = "{}"; + confluenceSearchResults = objectMapper.readValue(state, ConfluenceSearchResults.class); + } + + @Test + public void testConstructor() { + assertNotNull(confluenceSearchResults); + + } + + +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/IssueBeanTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/IssueBeanTest.java deleted file mode 100644 index 3b3216a03a..0000000000 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/IssueBeanTest.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - */ - -package org.opensearch.dataprepper.plugins.source.confluence.models; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.dataprepper.plugins.source.confluence.models.IssueBean; - -import java.util.HashMap; -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.KEY; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.NAME; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.PROJECT; - -@ExtendWith(MockitoExtension.class) -public class IssueBeanTest { - - @Mock - private Map renderedFieldsTestObject; - - @Mock - private Map propertiesTestObject; - - @Mock - private Map namesTestObject; - - @Mock - private Map fieldsTestObject; - - private IssueBean issueBean; - - @BeforeEach - void setup() { - issueBean = new IssueBean(); - } - - @Test - public void testInitialization() { - assertNotNull(issueBean); - } - - @Test - public void testNull() { - assertNull(issueBean.getExpand()); - assertNull(issueBean.getId()); - assertNull(issueBean.getSelf()); - assertNull(issueBean.getKey()); - assertNull(issueBean.getRenderedFields()); - assertNull(issueBean.getProperties()); - assertNull(issueBean.getNames()); - assertNull(issueBean.getFields()); - } - - @Test - void testNullCases() { - assertNull(issueBean.getProject()); - assertNull(issueBean.getProjectName()); - assertEquals(issueBean.getUpdatedTimeMillis(), 0); - } - - @Test - void testGivenDateField() { - Map fieldsTestObject = new HashMap<>(); - fieldsTestObject.put("created", "2024-07-06T21:12:23.437-0700"); - fieldsTestObject.put("updated", "2022-07-06T21:12:23.106-0700"); - issueBean.setFields(fieldsTestObject); - assertEquals(issueBean.getCreatedTimeMillis(), 1720325543000L); - assertEquals(issueBean.getUpdatedTimeMillis(), 1657167143000L); - } - - @Test - public void testStringSettersAndGetters() { - String self = "selfTest"; - String key = "keyTest"; - String id = "idTest"; - String expand = "expandTest"; - - issueBean.setExpand(expand); - assertEquals(issueBean.getExpand(), expand); - issueBean.setId(id); - assertEquals(issueBean.getId(), id); - issueBean.setSelf(self); - assertEquals(issueBean.getSelf(), self); - issueBean.setKey(key); - assertEquals(issueBean.getKey(), key); - } - - @Test - public void testMapSettersAndGetters() { - - issueBean.setRenderedFields(renderedFieldsTestObject); - assertEquals(issueBean.getRenderedFields(), renderedFieldsTestObject); - issueBean.setProperties(propertiesTestObject); - assertEquals(issueBean.getProperties(), propertiesTestObject); - issueBean.setNames(namesTestObject); - assertEquals(issueBean.getNames(), namesTestObject); - issueBean.setFields(fieldsTestObject); - assertEquals(issueBean.getFields(), fieldsTestObject); - } - - @Test - public void testFieldPropertyGetters() { - Map fieldsTestObject = new HashMap<>(); - Map projectTestObject = new HashMap<>(); - String projectName = "name of project"; - String projectKey = "PROJKEY"; - projectTestObject.put(KEY, projectKey); - projectTestObject.put(NAME, projectName); - fieldsTestObject.put(PROJECT, projectTestObject); - - issueBean.setFields(fieldsTestObject); - assertEquals(projectKey, issueBean.getProject()); - assertEquals(projectName, issueBean.getProjectName()); - } - -} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/SearchResultsTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/SearchResultsTest.java deleted file mode 100644 index 44321d2634..0000000000 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/SearchResultsTest.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - */ - -package org.opensearch.dataprepper.plugins.source.confluence.models; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.dataprepper.plugins.source.confluence.models.IssueBean; -import org.opensearch.dataprepper.plugins.source.confluence.models.SearchResults; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; - -@ExtendWith(MockitoExtension.class) -public class SearchResultsTest { - - private SearchResults searchResults; - private final ObjectMapper objectMapper = new ObjectMapper(); - - @BeforeEach - public void setUp() throws JsonProcessingException { - String state = "{}"; - searchResults = objectMapper.readValue(state, SearchResults.class); - } - - @Test - public void testConstructor() { - assertNotNull(searchResults); - - assertNull(searchResults.getExpand()); - assertNull(searchResults.getStartAt()); - assertNull(searchResults.getMaxResults()); - assertNull(searchResults.getTotal()); - assertNull(searchResults.getIssues()); - } - - @Test - public void testGetters() throws JsonProcessingException { - String expand = "expandTest"; - Integer startAt = 1; - Integer maxResults = 100; - Integer total = 10; - List testIssues = new ArrayList<>(); - IssueBean issue1 = new IssueBean(); - IssueBean issue2 = new IssueBean(); - issue1.setId("issue 1"); - issue2.setId("issue 2"); - testIssues.add(issue1); - testIssues.add(issue2); - - - Map searchResultsMap = new HashMap<>(); - searchResultsMap.put("expand", expand); - searchResultsMap.put("startAt", startAt); - searchResultsMap.put("maxResults", maxResults); - searchResultsMap.put("total", total); - searchResultsMap.put("issues", testIssues); - - String jsonString = objectMapper.writeValueAsString(searchResultsMap); - - searchResults = objectMapper.readValue(jsonString, SearchResults.class); - - assertEquals(searchResults.getExpand(), expand); - assertEquals(searchResults.getStartAt(), startAt); - assertEquals(searchResults.getMaxResults(), maxResults); - assertEquals(searchResults.getTotal(), total); - - List returnedIssues = searchResults.getIssues(); - assertNotNull(returnedIssues); - assertEquals(testIssues.size(), returnedIssues.size()); - - for (int i = 0; i < testIssues.size(); i++) { - IssueBean originalIssue = testIssues.get(i); - IssueBean returnedIssue = returnedIssues.get(i); - - assertEquals(originalIssue.getId(), returnedIssue.getId()); - } - } - - -} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClientTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClientTest.java index 58863e85e5..27a0885208 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClientTest.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClientTest.java @@ -19,11 +19,12 @@ import org.junit.jupiter.params.provider.ValueSource; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceServiceTest; import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceSourceConfig; import org.opensearch.dataprepper.plugins.source.confluence.exception.BadRequestException; import org.opensearch.dataprepper.plugins.source.confluence.exception.UnAuthorizedException; -import org.opensearch.dataprepper.plugins.source.confluence.models.SearchResults; +import org.opensearch.dataprepper.plugins.source.confluence.models.ConfluenceSearchResults; import org.opensearch.dataprepper.plugins.source.confluence.rest.auth.ConfluenceAuthConfig; import org.opensearch.dataprepper.plugins.source.confluence.rest.auth.ConfluenceAuthFactory; import org.springframework.http.HttpStatus; @@ -58,6 +59,8 @@ public class ConfluenceRestClientTest { @Mock private ConfluenceAuthConfig authConfig; + private final PluginMetrics pluginMetrics = PluginMetrics.fromNames("jiraRestClient", "aws"); + private static Stream provideHttpStatusCodesWithExceptionClass() { return Stream.of( Arguments.of(HttpStatus.FORBIDDEN, UnAuthorizedException.class), @@ -74,24 +77,24 @@ public void testFetchingJiraIssue(String configFileName) { doReturn(new ResponseEntity<>(exampleTicketResponse, HttpStatus.OK)).when(restTemplate).getForEntity(any(URI.class), any(Class.class)); ConfluenceSourceConfig confluenceSourceConfig = ConfluenceServiceTest.createJiraConfigurationFromYaml(configFileName); ConfluenceAuthConfig authConfig = new ConfluenceAuthFactory(confluenceSourceConfig).getObject(); - ConfluenceRestClient confluenceRestClient = new ConfluenceRestClient(restTemplate, authConfig); - String ticketDetails = confluenceRestClient.getIssue("key"); + ConfluenceRestClient confluenceRestClient = new ConfluenceRestClient(restTemplate, authConfig, pluginMetrics); + String ticketDetails = confluenceRestClient.getContent("key"); assertEquals(exampleTicketResponse, ticketDetails); } @ParameterizedTest @MethodSource("provideHttpStatusCodesWithExceptionClass") void testInvokeRestApiTokenExpired(HttpStatus statusCode, Class expectedExceptionType) { - ConfluenceRestClient confluenceRestClient = new ConfluenceRestClient(restTemplate, authConfig); + ConfluenceRestClient confluenceRestClient = new ConfluenceRestClient(restTemplate, authConfig, pluginMetrics); confluenceRestClient.setSleepTimeMultiplier(1); when(authConfig.getUrl()).thenReturn("https://example.com/rest/api/2/issue/key"); when(restTemplate.getForEntity(any(URI.class), any(Class.class))).thenThrow(new HttpClientErrorException(statusCode)); - assertThrows(expectedExceptionType, () -> confluenceRestClient.getIssue("key")); + assertThrows(expectedExceptionType, () -> confluenceRestClient.getContent("key")); } @Test void testInvokeRestApiTokenExpiredInterruptException() throws InterruptedException { - ConfluenceRestClient confluenceRestClient = new ConfluenceRestClient(restTemplate, authConfig); + ConfluenceRestClient confluenceRestClient = new ConfluenceRestClient(restTemplate, authConfig, pluginMetrics); when(authConfig.getUrl()).thenReturn("https://example.com/rest/api/2/issue/key"); when(restTemplate.getForEntity(any(URI.class), any(Class.class))).thenThrow(new HttpClientErrorException(HttpStatus.TOO_MANY_REQUESTS)); confluenceRestClient.setSleepTimeMultiplier(100000); @@ -99,7 +102,7 @@ void testInvokeRestApiTokenExpiredInterruptException() throws InterruptedExcepti Thread testThread = new Thread(() -> { assertThrows(InterruptedException.class, () -> { try { - confluenceRestClient.getIssue("key"); + confluenceRestClient.getContent("key"); } catch (Exception e) { throw new RuntimeException(e); } @@ -111,40 +114,40 @@ void testInvokeRestApiTokenExpiredInterruptException() throws InterruptedExcepti } @Test - public void testGetAllIssuesOauth2() throws JsonProcessingException { + public void testGetAllContentOauth2() throws JsonProcessingException { List issueType = new ArrayList<>(); List issueStatus = new ArrayList<>(); List projectKey = new ArrayList<>(); issueType.add("Task"); ConfluenceSourceConfig confluenceSourceConfig = ConfluenceServiceTest.createJiraConfiguration(OAUTH2, issueType, issueStatus, projectKey); - ConfluenceRestClient confluenceRestClient = new ConfluenceRestClient(restTemplate, authConfig); - SearchResults mockSearchResults = mock(SearchResults.class); + ConfluenceRestClient confluenceRestClient = new ConfluenceRestClient(restTemplate, authConfig, pluginMetrics); + ConfluenceSearchResults mockConfluenceSearchResults = mock(ConfluenceSearchResults.class); doReturn("http://mock-service.jira.com/").when(authConfig).getUrl(); - doReturn(new ResponseEntity<>(mockSearchResults, HttpStatus.OK)).when(restTemplate).getForEntity(any(URI.class), any(Class.class)); - SearchResults results = confluenceRestClient.getAllIssues(jql, 0, confluenceSourceConfig); + doReturn(new ResponseEntity<>(mockConfluenceSearchResults, HttpStatus.OK)).when(restTemplate).getForEntity(any(URI.class), any(Class.class)); + ConfluenceSearchResults results = confluenceRestClient.getAllContent(jql, 0); assertNotNull(results); } @Test - public void testGetAllIssuesBasic() throws JsonProcessingException { + public void testGetAllContentBasic() throws JsonProcessingException { List issueType = new ArrayList<>(); List issueStatus = new ArrayList<>(); List projectKey = new ArrayList<>(); issueType.add("Task"); ConfluenceSourceConfig confluenceSourceConfig = ConfluenceServiceTest.createJiraConfiguration(BASIC, issueType, issueStatus, projectKey); - ConfluenceRestClient confluenceRestClient = new ConfluenceRestClient(restTemplate, authConfig); - SearchResults mockSearchResults = mock(SearchResults.class); + ConfluenceRestClient confluenceRestClient = new ConfluenceRestClient(restTemplate, authConfig, pluginMetrics); + ConfluenceSearchResults mockConfluenceSearchResults = mock(ConfluenceSearchResults.class); when(authConfig.getUrl()).thenReturn("https://example.com/"); - doReturn(new ResponseEntity<>(mockSearchResults, HttpStatus.OK)).when(restTemplate).getForEntity(any(URI.class), any(Class.class)); - SearchResults results = confluenceRestClient.getAllIssues(jql, 0, confluenceSourceConfig); + doReturn(new ResponseEntity<>(mockConfluenceSearchResults, HttpStatus.OK)).when(restTemplate).getForEntity(any(URI.class), any(Class.class)); + ConfluenceSearchResults results = confluenceRestClient.getAllContent(jql, 0); assertNotNull(results); } @Test public void testRestApiAddressValidation() throws JsonProcessingException { when(authConfig.getUrl()).thenReturn("https://224.0.0.1/"); - ConfluenceRestClient confluenceRestClient = new ConfluenceRestClient(restTemplate, authConfig); - assertThrows(BadRequestException.class, () -> confluenceRestClient.getIssue("TEST-1")); + ConfluenceRestClient confluenceRestClient = new ConfluenceRestClient(restTemplate, authConfig, pluginMetrics); + assertThrows(BadRequestException.class, () -> confluenceRestClient.getContent("TEST-1")); } } diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/utils/AddressValidationTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/utils/AddressValidationTest.java index 0077129427..862d01878e 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/utils/AddressValidationTest.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/utils/AddressValidationTest.java @@ -12,9 +12,6 @@ import org.junit.jupiter.api.Test; import org.opensearch.dataprepper.plugins.source.confluence.exception.BadRequestException; -import org.opensearch.dataprepper.plugins.source.confluence.utils.AddressValidation; -import org.opensearch.dataprepper.plugins.source.confluence.utils.Constants; -import org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants; import java.net.InetAddress; import java.net.MalformedURLException; @@ -31,7 +28,7 @@ public class AddressValidationTest { void testInstanceCreation() { assertNotNull(new AddressValidation()); assertNotNull(new Constants()); - assertNotNull(new JqlConstants()); + assertNotNull(new CqlConstants()); } @Test From 378c41ffedf237f6049602e415cc4e44215ffb58 Mon Sep 17 00:00:00 2001 From: Santhosh Gandhe <1909520+san81@users.noreply.github.com> Date: Mon, 3 Feb 2025 12:42:35 -0800 Subject: [PATCH 04/14] Saving page content as text Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com> --- .../confluence-source/build.gradle | 1 + .../source/confluence/ConfluenceClient.java | 7 +- .../source/confluence/ConfluenceService.java | 15 +- .../utils/HtmlToTextConversionUtil.java | 53 ++++ .../utils/HtmlToTextConversionUtilTest.java | 276 ++++++++++++++++++ 5 files changed, 342 insertions(+), 10 deletions(-) create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/HtmlToTextConversionUtil.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/utils/HtmlToTextConversionUtilTest.java diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/build.gradle b/data-prepper-plugins/saas-source-plugins/confluence-source/build.gradle index 1cf3854ab4..b59ab8d251 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/build.gradle +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/build.gradle @@ -15,6 +15,7 @@ dependencies { implementation 'com.fasterxml.jackson.core:jackson-core' implementation 'com.fasterxml.jackson.core:jackson-databind' implementation 'javax.inject:javax.inject:1' + implementation 'org.jsoup:jsoup:1.18.3' implementation("org.springframework:spring-web:${libs.versions.spring.get()}") implementation 'org.projectlombok:lombok:1.18.30' diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceClient.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceClient.java index f4dc7cfe4f..b5ee7a1446 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceClient.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceClient.java @@ -13,6 +13,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.annotations.VisibleForTesting; import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSet; import org.opensearch.dataprepper.model.buffer.Buffer; @@ -20,6 +21,7 @@ import org.opensearch.dataprepper.model.event.EventType; import org.opensearch.dataprepper.model.event.JacksonEvent; import org.opensearch.dataprepper.model.record.Record; +import org.opensearch.dataprepper.plugins.source.confluence.utils.HtmlToTextConversionUtil; import org.opensearch.dataprepper.plugins.source.source_crawler.base.CrawlerClient; import org.opensearch.dataprepper.plugins.source.source_crawler.base.CrawlerSourceConfig; import org.opensearch.dataprepper.plugins.source.source_crawler.base.PluginExecutorServiceProvider; @@ -117,8 +119,9 @@ public void executePartition(SaasWorkerProgressState state, .map(CompletableFuture::join) .map(contentJson -> { try { - return objectMapper.readValue(contentJson, new TypeReference<>() { + ObjectNode contentJsonObj = objectMapper.readValue(contentJson, new TypeReference<>() { }); + return HtmlToTextConversionUtil.convertHtmlToText(contentJsonObj, "body/view/value"); } catch (JsonProcessingException e) { throw new RuntimeException(e); } @@ -127,7 +130,7 @@ public void executePartition(SaasWorkerProgressState state, .withEventType(eventType) .withData(t) .build()) - .map(event -> new Record<>(event)) + .map(Record::new) .collect(Collectors.toList()); try { diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java index 457c98c40c..b71342089f 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java @@ -117,9 +117,8 @@ private void searchForNewContentAndAddToQueue(ConfluenceSourceConfig configurati * @param itemInfoQueue Item info queue. */ private void addItemsToQueue(List issueList, Queue itemInfoQueue) { - issueList.forEach(issue -> { - itemInfoQueue.add(ConfluenceItemInfo.builder().withEventTime(Instant.now()).withIssueBean(issue).build()); - }); + issueList.forEach(issue -> itemInfoQueue.add(ConfluenceItemInfo.builder() + .withEventTime(Instant.now()).withIssueBean(issue).build())); } @@ -173,7 +172,7 @@ private void validateSpaceFilters(ConfluenceSourceConfig configuration) { log.trace("Validating project filters"); List badFilters = new ArrayList<>(); Set includedProjects = new HashSet<>(); - List includedAndExcludedProjects = new ArrayList<>(); + List includedAndExcludedSpaces = new ArrayList<>(); Pattern regex = Pattern.compile("[^A-Z0-9]"); ConfluenceConfigHelper.getSpacesNameIncludeFilter(configuration).forEach(projectFilter -> { Matcher matcher = regex.matcher(projectFilter); @@ -185,7 +184,7 @@ private void validateSpaceFilters(ConfluenceSourceConfig configuration) { ConfluenceConfigHelper.getSpacesNameExcludeFilter(configuration).forEach(projectFilter -> { Matcher matcher = regex.matcher(projectFilter); if (includedProjects.contains(projectFilter)) { - includedAndExcludedProjects.add(projectFilter); + includedAndExcludedSpaces.add(projectFilter); } if (matcher.find() || projectFilter.length() <= 1 || projectFilter.length() > 10) { badFilters.add(projectFilter); @@ -198,9 +197,9 @@ private void validateSpaceFilters(ConfluenceSourceConfig configuration) { "Invalid Space key found in filter configuration for " + filters); } - if (!includedAndExcludedProjects.isEmpty()) { - String filters = String.join("\"" + includedAndExcludedProjects + "\"", ", "); - log.error("One or more Space keys found in both include and exclude: {}", includedAndExcludedProjects); + if (!includedAndExcludedSpaces.isEmpty()) { + String filters = String.join("\"" + includedAndExcludedSpaces + "\"", ", "); + log.error("One or more Space keys found in both include and exclude: {}", includedAndExcludedSpaces); throw new BadRequestException("Bad request exception occurred " + "Space filters is invalid because the following space are listed in both include and exclude" + filters); diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/HtmlToTextConversionUtil.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/HtmlToTextConversionUtil.java new file mode 100644 index 0000000000..3d34dbf8e3 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/HtmlToTextConversionUtil.java @@ -0,0 +1,53 @@ +package org.opensearch.dataprepper.plugins.source.confluence.utils; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; + +import java.util.Optional; + +public class HtmlToTextConversionUtil { + + public static JsonNode convertHtmlToText(ObjectNode jsonObject, String path) { + Optional valueAtGivenPath = getValueAtGivenPath(jsonObject, path); + if (valueAtGivenPath.isPresent()) { + String html = valueAtGivenPath.get().textValue(); + String txtBody = convertHtmlToText(html); + setValueAtGivenPath(jsonObject, path, txtBody); + } + return jsonObject; + } + + public static void setValueAtGivenPath(ObjectNode jsonObject, String path, String value) { + String[] keys = path.split("/"); + JsonNode current = jsonObject; + for (int i = 0; i < keys.length - 1; i++) { + current = current.get(keys[i]); + } + ((ObjectNode) current).put(keys[keys.length - 1], value); + } + + public static Optional getValueAtGivenPath(ObjectNode jsonObject, String path) { + try { + String[] keys = path.split("/"); + ObjectNode current = jsonObject; + for (int i = 0; i < keys.length - 1; i++) { + current = (ObjectNode) current.get(keys[i]); + } + return Optional.of(current.get(keys[keys.length - 1])); + } catch (Exception e) { + return Optional.empty(); + } + } + + public static String convertHtmlToText(String html) { + if (html == null || html.isEmpty()) { + return ""; + } + Document document = Jsoup.parse(html); + // Remove scripts and style elements + document.select("script, style").remove(); + return document.text(); + } +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/utils/HtmlToTextConversionUtilTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/utils/HtmlToTextConversionUtilTest.java new file mode 100644 index 0000000000..23d6421aea --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/utils/HtmlToTextConversionUtilTest.java @@ -0,0 +1,276 @@ +package org.opensearch.dataprepper.plugins.source.confluence.utils; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.Optional; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class HtmlToTextConversionUtilTest { + + private ObjectMapper objectMapper; + private ObjectNode jsonObject; + + private static Stream provideDifferentTypeValues() { + return Stream.of( + Arguments.of(42, "number"), + Arguments.of(true, "boolean"), + Arguments.of(3.14, "number"), + Arguments.of("string value", "string") + ); + } + + @BeforeEach + void setUp() { + objectMapper = new ObjectMapper(); + jsonObject = objectMapper.createObjectNode(); + } + + @Test + void convertHtmlToText_WithValidHtmlAndPath_ShouldConvertSuccessfully() { + // Arrange + jsonObject.put("content", "

Hello World

"); + String path = "content"; + + // Act + JsonNode result = HtmlToTextConversionUtil.convertHtmlToText(jsonObject, path); + + // Assert + assertTrue(result.has("content")); + assertEquals("Hello World", result.get("content").textValue()); + } + + @Test + void convertHtmlToText_WithNestedPath_ShouldConvertSuccessfully() { + // Arrange + ObjectNode nestedNode = objectMapper.createObjectNode(); + nestedNode.put("body", "

Hello World

"); + jsonObject.set("content", nestedNode); + String path = "content/body"; + + // Act + JsonNode result = HtmlToTextConversionUtil.convertHtmlToText(jsonObject, path); + + // Assert + assertTrue(result.has("content")); + assertTrue(result.get("content").has("body")); + assertEquals("Hello World", result.get("content").get("body").textValue()); + } + + @Test + void setValueAtGivenPath_WithSimplePath_ShouldSetValue() { + // Arrange + String path = "title"; + String value = "Test Title"; + + // Act + HtmlToTextConversionUtil.setValueAtGivenPath(jsonObject, path, value); + + // Assert + assertEquals(value, jsonObject.get("title").asText()); + } + + @Test + void setValueAtGivenPath_WithNestedPath_ShouldSetValue() { + // Arrange + jsonObject.putObject("content").putObject("body"); + String path = "content/body/text"; + String value = "Test Content"; + + // Act + HtmlToTextConversionUtil.setValueAtGivenPath(jsonObject, path, value); + + // Assert + assertEquals(value, jsonObject.get("content").get("body").get("text").asText()); + } + + @Test + void convertHtmlToText_WithNullHtml_ShouldReturnEmptyString() { + // Act + String result = HtmlToTextConversionUtil.convertHtmlToText(null); + + // Assert + assertEquals("", result); + } + + @Test + void convertHtmlToText_WithEmptyHtml_ShouldReturnEmptyString() { + // Act + String result = HtmlToTextConversionUtil.convertHtmlToText(""); + + // Assert + assertEquals("", result); + } + + @Test + void convertHtmlToText_WithComplexHtml_ShouldConvertToPlainText() { + // Arrange + String html = "

Title

This is a test paragraph

"; + + // Act + String result = HtmlToTextConversionUtil.convertHtmlToText(html); + + // Assert + assertEquals("Title This is a test paragraph", result.trim()); + } + + @Test + void convertHtmlToText_WithInvalidPath_ShouldReturnOriginalJson() { + // Arrange + jsonObject.put("content", "

Hello World

"); + String invalidPath = "invalid.path"; + + // Act + JsonNode result = HtmlToTextConversionUtil.convertHtmlToText(jsonObject, invalidPath); + + // Assert + assertEquals(jsonObject, result); + } + + @Test + void getValueAtGivenPath_WithSimplePath_ShouldReturnValue() { + // Arrange + String expectedValue = "test value"; + jsonObject.put("key", expectedValue); + + // Act + Optional result = HtmlToTextConversionUtil.getValueAtGivenPath(jsonObject, "key"); + + // Assert + assertTrue(result.isPresent()); + assertEquals(expectedValue, result.get().asText()); + } + + @Test + void getValueAtGivenPath_WithNestedPath_ShouldReturnValue() { + // Arrange + String expectedValue = "nested value"; + ObjectNode nestedNode = jsonObject.putObject("parent"); + nestedNode.put("child", expectedValue); + + // Act + Optional result = HtmlToTextConversionUtil.getValueAtGivenPath(jsonObject, "parent/child"); + + // Assert + assertTrue(result.isPresent()); + assertEquals(expectedValue, result.get().asText()); + } + + @Test + void getValueAtGivenPath_WithDeeplyNestedPath_ShouldReturnValue() { + // Arrange + String expectedValue = "deeply nested value"; + ObjectNode level1 = jsonObject.putObject("level1"); + ObjectNode level2 = level1.putObject("level2"); + level2.put("level3", expectedValue); + + // Act + Optional result = HtmlToTextConversionUtil.getValueAtGivenPath( + jsonObject, "level1/level2/level3"); + + // Assert + assertTrue(result.isPresent()); + assertEquals(expectedValue, result.get().asText()); + } + + @Test + void getValueAtGivenPath_WithNonExistentPath_ShouldReturnEmpty() { + // Act + Optional result = HtmlToTextConversionUtil.getValueAtGivenPath( + jsonObject, "nonexistent/path"); + + // Assert + assertFalse(result.isPresent()); + } + + @Test + void getValueAtGivenPath_WithNullPath_ShouldReturnEmpty() { + // Act + Optional result = HtmlToTextConversionUtil.getValueAtGivenPath(jsonObject, null); + + // Assert + assertFalse(result.isPresent()); + } + + @Test + void getValueAtGivenPath_WithEmptyPath_ShouldReturnEmpty() { + // Act + Optional result = HtmlToTextConversionUtil.getValueAtGivenPath(jsonObject, ""); + + // Assert + assertFalse(result.isPresent()); + } + + @ParameterizedTest + @MethodSource("provideDifferentTypeValues") + void getValueAtGivenPath_WithDifferentTypes_ShouldReturnCorrectValue(Object value, String expectedType) { + // Arrange + if (value instanceof Integer) { + jsonObject.put("key", (Integer) value); + } else if (value instanceof Boolean) { + jsonObject.put("key", (Boolean) value); + } else if (value instanceof Double) { + jsonObject.put("key", (Double) value); + } else { + jsonObject.put("key", String.valueOf(value)); + } + + // Act + Optional result = HtmlToTextConversionUtil.getValueAtGivenPath(jsonObject, "key"); + + // Assert + assertTrue(result.isPresent()); + assertEquals(value.toString(), result.get().asText()); + } + + @Test + void getValueAtGivenPath_WithInvalidIntermediatePath_ShouldReturnEmpty() { + // Arrange + jsonObject.put("key", "value"); + + // Act + Optional result = HtmlToTextConversionUtil.getValueAtGivenPath( + jsonObject, "nonexistent/key/child"); + + // Assert + assertFalse(result.isPresent()); + } + + @Test + void getValueAtGivenPath_WithNullIntermediateNode_ShouldReturnEmpty() { + // Arrange + ObjectNode parentNode = jsonObject.putObject("parent"); + parentNode.putNull("child"); + + // Act + Optional result = HtmlToTextConversionUtil.getValueAtGivenPath( + jsonObject, "parent/child/grandchild"); + + // Assert + assertFalse(result.isPresent()); + } + + @Test + void getValueAtGivenPath_WithArrayNode_ShouldReturnEmpty() { + // Arrange + jsonObject.putArray("array").add("value"); + + // Act + Optional result = HtmlToTextConversionUtil.getValueAtGivenPath( + jsonObject, "array/0"); + + // Assert + assertFalse(result.isPresent()); + } +} + From d9a98d06e6c6b881368963147facf08b8ba81a86 Mon Sep 17 00:00:00 2001 From: Santhosh Gandhe <1909520+san81@users.noreply.github.com> Date: Mon, 3 Feb 2025 13:00:32 -0800 Subject: [PATCH 05/14] injectable plugin metrics Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com> --- .../plugins/source/confluence/ConfluenceService.java | 7 ++++--- .../source/confluence/ConfluenceIteratorTest.java | 4 +++- .../source/confluence/ConfluenceServiceTest.java | 12 +++++++----- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java index b71342089f..877960c634 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java @@ -62,13 +62,14 @@ public class ConfluenceService { private final ConfluenceSourceConfig confluenceSourceConfig; private final ConfluenceRestClient confluenceRestClient; private final Counter searchResultsFoundCounter; - private final PluginMetrics jiraPluginMetrics = PluginMetrics.fromNames("jiraService", "aws"); - public ConfluenceService(ConfluenceSourceConfig confluenceSourceConfig, ConfluenceRestClient confluenceRestClient) { + public ConfluenceService(ConfluenceSourceConfig confluenceSourceConfig, + ConfluenceRestClient confluenceRestClient, + PluginMetrics pluginMetrics) { this.confluenceSourceConfig = confluenceSourceConfig; this.confluenceRestClient = confluenceRestClient; - this.searchResultsFoundCounter = jiraPluginMetrics.counter(SEARCH_RESULTS_FOUND); + this.searchResultsFoundCounter = pluginMetrics.counter(SEARCH_RESULTS_FOUND); } /** diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceIteratorTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceIteratorTest.java index cc2bfde044..3dfbe0be6e 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceIteratorTest.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceIteratorTest.java @@ -15,6 +15,7 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.plugins.source.confluence.models.ConfluenceSearchResults; import org.opensearch.dataprepper.plugins.source.confluence.rest.ConfluenceRestClient; import org.opensearch.dataprepper.plugins.source.source_crawler.base.PluginExecutorServiceProvider; @@ -41,11 +42,12 @@ public class ConfluenceIteratorTest { @Mock private ConfluenceSourceConfig confluenceSourceConfig; private ConfluenceIterator confluenceIterator; + private final PluginMetrics pluginMetrics = PluginMetrics.fromNames("confluenceService", "aws"); private final PluginExecutorServiceProvider executorServiceProvider = new PluginExecutorServiceProvider(); @BeforeEach void setUp() { - confluenceService = spy(new ConfluenceService(confluenceSourceConfig, confluenceRestClient)); + confluenceService = spy(new ConfluenceService(confluenceSourceConfig, confluenceRestClient, pluginMetrics)); } public ConfluenceIterator createObjectUnderTest() { diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceServiceTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceServiceTest.java index df4b9d32be..d966b2e3ad 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceServiceTest.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceServiceTest.java @@ -18,6 +18,7 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.plugin.PluginConfigVariable; import org.opensearch.dataprepper.plugins.source.confluence.configuration.Oauth2Config; import org.opensearch.dataprepper.plugins.source.confluence.exception.BadRequestException; @@ -74,6 +75,7 @@ public class ConfluenceServiceTest { @Mock private ConfluenceRestClient confluenceRestClient; private final PluginExecutorServiceProvider executorServiceProvider = new PluginExecutorServiceProvider(); + private final PluginMetrics pluginMetrics = PluginMetrics.fromNames("confluenceService", "aws"); private static InputStream getResourceAsStream(String resourceName) { InputStream inputStream = Thread.currentThread().getContextClassLoader().getResourceAsStream(resourceName); @@ -176,7 +178,7 @@ void testJiraServiceInitialization() throws JsonProcessingException { List issueStatus = new ArrayList<>(); List projectKey = new ArrayList<>(); ConfluenceSourceConfig confluenceSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey); - ConfluenceService confluenceService = new ConfluenceService(confluenceSourceConfig, confluenceRestClient); + ConfluenceService confluenceService = new ConfluenceService(confluenceSourceConfig, confluenceRestClient, pluginMetrics); assertNotNull(confluenceService); when(confluenceRestClient.getContent(anyString())).thenReturn("test String"); assertNotNull(confluenceService.getContent("test Key")); @@ -191,7 +193,7 @@ public void testGetPages() throws JsonProcessingException { issueStatus.add("Done"); projectKey.add("KAN"); ConfluenceSourceConfig confluenceSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey); - ConfluenceService confluenceService = spy(new ConfluenceService(confluenceSourceConfig, confluenceRestClient)); + ConfluenceService confluenceService = spy(new ConfluenceService(confluenceSourceConfig, confluenceRestClient, pluginMetrics)); List mockIssues = new ArrayList<>(); ConfluenceItem issue1 = createConfluenceItemBean(false, false); mockIssues.add(issue1); @@ -217,7 +219,7 @@ public void buildIssueItemInfoMultipleFutureThreads() throws JsonProcessingExcep List projectKey = new ArrayList<>(); issueType.add("Task"); ConfluenceSourceConfig confluenceSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey); - ConfluenceService confluenceService = spy(new ConfluenceService(confluenceSourceConfig, confluenceRestClient)); + ConfluenceService confluenceService = spy(new ConfluenceService(confluenceSourceConfig, confluenceRestClient, pluginMetrics)); List mockIssues = new ArrayList<>(); for (int i = 0; i < 50; i++) { ConfluenceItem issue1 = createConfluenceItemBean(false, false); @@ -248,7 +250,7 @@ public void testBadProjectKeys() throws JsonProcessingException { projectKey.add("AAAAAAAAAAAAAA"); ConfluenceSourceConfig confluenceSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey); - ConfluenceService confluenceService = new ConfluenceService(confluenceSourceConfig, confluenceRestClient); + ConfluenceService confluenceService = new ConfluenceService(confluenceSourceConfig, confluenceRestClient, pluginMetrics); Instant timestamp = Instant.ofEpochSecond(0); Queue itemInfoQueue = new ConcurrentLinkedQueue<>(); @@ -263,7 +265,7 @@ public void testGetPagesException() throws JsonProcessingException { List projectKey = new ArrayList<>(); issueType.add("Task"); ConfluenceSourceConfig confluenceSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey); - ConfluenceService confluenceService = spy(new ConfluenceService(confluenceSourceConfig, confluenceRestClient)); + ConfluenceService confluenceService = spy(new ConfluenceService(confluenceSourceConfig, confluenceRestClient, pluginMetrics)); doThrow(RuntimeException.class).when(confluenceRestClient).getAllContent(any(StringBuilder.class), anyInt()); From 04ac7c042d3e541b79f0f5d60a18a6f506d6d754 Mon Sep 17 00:00:00 2001 From: Santhosh Gandhe <1909520+san81@users.noreply.github.com> Date: Mon, 3 Feb 2025 14:43:19 -0800 Subject: [PATCH 06/14] Introduced atlassian common module Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com> --- .../configuration/AtlassianSourceConfig.java | 58 ++++++ .../configuration/AuthenticationConfig.java | 46 +++++ .../atlassian/configuration/BasicConfig.java | 29 +++ .../atlassian/configuration/Oauth2Config.java | 36 ++++ .../atlassian/rest/AtlassianRestClient.java | 79 ++++++++ .../atlassian/rest/BasicAuthInterceptor.java | 42 ++++ .../rest/CustomRestTemplateConfig.java | 42 ++++ .../rest/OAuth2RequestInterceptor.java | 36 ++++ .../rest/auth/AtlassianAuthConfig.java | 34 ++++ .../rest/auth/AtlassianAuthFactory.java | 41 ++++ .../rest/auth/AtlassianBasicAuthConfig.java | 45 +++++ .../rest/auth/AtlassianOauthConfig.java | 183 ++++++++++++++++++ .../source/atlassian/utils/Constants.java | 23 +++ .../source/confluence/ConfluenceService.java | 2 +- .../confluence/rest/ConfluenceRestClient.java | 6 +- .../rest/auth/ConfluenceOauthConfig.java | 2 +- .../utils/ConfluenceConfigHelper.java | 1 + .../exception/BadRequestException.java | 2 +- .../exception/UnAuthorizedException.java | 2 +- .../utils/AddressValidation.java | 4 +- .../exception/BadRequestExceptionTest.java | 3 +- .../exception/UnAuthorizedExceptionTest.java | 3 +- .../utils/AddressValidationTest.java | 6 +- settings.gradle | 1 + 24 files changed, 709 insertions(+), 17 deletions(-) create mode 100644 data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/configuration/AtlassianSourceConfig.java create mode 100644 data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/configuration/AuthenticationConfig.java create mode 100644 data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/configuration/BasicConfig.java create mode 100644 data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/configuration/Oauth2Config.java create mode 100644 data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/AtlassianRestClient.java create mode 100644 data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/BasicAuthInterceptor.java create mode 100644 data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/CustomRestTemplateConfig.java create mode 100644 data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/OAuth2RequestInterceptor.java create mode 100644 data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianAuthConfig.java create mode 100644 data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianAuthFactory.java create mode 100644 data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianBasicAuthConfig.java create mode 100644 data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianOauthConfig.java create mode 100644 data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/utils/Constants.java rename data-prepper-plugins/saas-source-plugins/{confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence => source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler}/exception/BadRequestException.java (89%) rename data-prepper-plugins/saas-source-plugins/{confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence => source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler}/exception/UnAuthorizedException.java (90%) rename data-prepper-plugins/saas-source-plugins/{confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence => source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler}/utils/AddressValidation.java (91%) rename data-prepper-plugins/saas-source-plugins/{confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence => source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler}/exception/BadRequestExceptionTest.java (91%) rename data-prepper-plugins/saas-source-plugins/{confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence => source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler}/exception/UnAuthorizedExceptionTest.java (91%) rename data-prepper-plugins/saas-source-plugins/{confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence => source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler}/utils/AddressValidationTest.java (93%) diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/configuration/AtlassianSourceConfig.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/configuration/AtlassianSourceConfig.java new file mode 100644 index 0000000000..448206469a --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/configuration/AtlassianSourceConfig.java @@ -0,0 +1,58 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.atlassian.configuration; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.Valid; +import lombok.Getter; +import org.opensearch.dataprepper.plugins.source.source_crawler.base.CrawlerSourceConfig; + +import java.util.List; + +@Getter +public class AtlassianSourceConfig implements CrawlerSourceConfig { + + private static final int DEFAULT_BATCH_SIZE = 50; + + /** + * Jira account url + */ + @JsonProperty("hosts") + protected List hosts; + + /** + * Authentication Config to Access Jira + */ + @JsonProperty("authentication") + @Valid + protected AuthenticationConfig authenticationConfig; + + /** + * Batch size for fetching tickets + */ + @JsonProperty("batch_size") + protected int batchSize = DEFAULT_BATCH_SIZE; + + + /** + * Boolean property indicating end to end acknowledgments state + */ + @JsonProperty("acknowledgments") + private boolean acknowledgments = false; + + public String getAccountUrl() { + return this.getHosts().get(0); + } + + public String getAuthType() { + return this.getAuthenticationConfig().getAuthType(); + } +} diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/configuration/AuthenticationConfig.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/configuration/AuthenticationConfig.java new file mode 100644 index 0000000000..0d1cf8b961 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/configuration/AuthenticationConfig.java @@ -0,0 +1,46 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.atlassian.configuration; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.Valid; +import jakarta.validation.constraints.AssertTrue; +import lombok.Getter; + +import static org.opensearch.dataprepper.plugins.source.atlassian.utils.Constants.BASIC; +import static org.opensearch.dataprepper.plugins.source.atlassian.utils.Constants.OAUTH2; + + +@Getter +public class AuthenticationConfig { + @JsonProperty("basic") + @Valid + private BasicConfig basicConfig; + + @JsonProperty("oauth2") + @Valid + private Oauth2Config oauth2Config; + + @AssertTrue(message = "Authentication config should have either basic or oauth2") + private boolean isValidAuthenticationConfig() { + boolean hasBasic = basicConfig != null; + boolean hasOauth = oauth2Config != null; + return hasBasic ^ hasOauth; + } + + public String getAuthType() { + if (basicConfig != null) { + return BASIC; + } else { + return OAUTH2; + } + } +} diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/configuration/BasicConfig.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/configuration/BasicConfig.java new file mode 100644 index 0000000000..a610be6e2a --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/configuration/BasicConfig.java @@ -0,0 +1,29 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.atlassian.configuration; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.constraints.AssertTrue; +import lombok.Getter; + +@Getter +public class BasicConfig { + @JsonProperty("username") + private String username; + + @JsonProperty("password") + private String password; + + @AssertTrue(message = "Username and Password are both required for Basic Auth") + private boolean isBasicConfigValid() { + return username != null && password != null; + } +} diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/configuration/Oauth2Config.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/configuration/Oauth2Config.java new file mode 100644 index 0000000000..812ef61afe --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/configuration/Oauth2Config.java @@ -0,0 +1,36 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.atlassian.configuration; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.constraints.AssertTrue; +import lombok.Getter; +import org.opensearch.dataprepper.model.plugin.PluginConfigVariable; + +@Getter +public class Oauth2Config { + @JsonProperty("client_id") + private String clientId; + + @JsonProperty("client_secret") + private String clientSecret; + + @JsonProperty("access_token") + private PluginConfigVariable accessToken; + + @JsonProperty("refresh_token") + private PluginConfigVariable refreshToken; + + @AssertTrue(message = "Client ID, Client Secret, Access Token, and Refresh Token are both required for Oauth2") + private boolean isOauth2ConfigValid() { + return clientId != null && clientSecret != null && accessToken != null && refreshToken != null; + } +} diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/AtlassianRestClient.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/AtlassianRestClient.java new file mode 100644 index 0000000000..91c2ae633a --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/AtlassianRestClient.java @@ -0,0 +1,79 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.atlassian.rest; + +import com.google.common.annotations.VisibleForTesting; +import lombok.extern.slf4j.Slf4j; +import org.opensearch.dataprepper.plugins.source.atlassian.rest.auth.AtlassianAuthConfig; +import org.opensearch.dataprepper.plugins.source.source_crawler.exception.BadRequestException; +import org.opensearch.dataprepper.plugins.source.source_crawler.exception.UnAuthorizedException; +import org.opensearch.dataprepper.plugins.source.source_crawler.utils.AddressValidation; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.RestTemplate; + +import java.net.URI; +import java.util.List; + +import static org.opensearch.dataprepper.logging.DataPrepperMarkers.NOISY; +import static org.opensearch.dataprepper.plugins.source.atlassian.utils.Constants.RETRY_ATTEMPT; + +@Slf4j +public class AtlassianRestClient { + + public static final List RETRY_ATTEMPT_SLEEP_TIME = List.of(1, 2, 5, 10, 20, 40); + private int sleepTimeMultiplier = 1000; + private final RestTemplate restTemplate; + private final AtlassianAuthConfig authConfig; + + public AtlassianRestClient(RestTemplate restTemplate, AtlassianAuthConfig authConfig) { + this.restTemplate = restTemplate; + this.authConfig = authConfig; + } + + + protected ResponseEntity invokeRestApi(URI uri, Class responseType) throws BadRequestException { + AddressValidation.validateInetAddress(AddressValidation.getInetAddress(uri.toString())); + int retryCount = 0; + while (retryCount < RETRY_ATTEMPT) { + try { + return restTemplate.getForEntity(uri, responseType); + } catch (HttpClientErrorException ex) { + HttpStatus statusCode = ex.getStatusCode(); + String statusMessage = ex.getMessage(); + log.error("An exception has occurred while getting response from Jira search API {}", ex.getMessage()); + if (statusCode == HttpStatus.FORBIDDEN) { + throw new UnAuthorizedException(statusMessage); + } else if (statusCode == HttpStatus.UNAUTHORIZED) { + log.error(NOISY, "Token expired. We will try to renew the tokens now", ex); + authConfig.renewCredentials(); + } else if (statusCode == HttpStatus.TOO_MANY_REQUESTS) { + log.error(NOISY, "Hitting API rate limit. Backing off with sleep timer.", ex); + } + try { + Thread.sleep((long) RETRY_ATTEMPT_SLEEP_TIME.get(retryCount) * sleepTimeMultiplier); + } catch (InterruptedException e) { + throw new RuntimeException("Sleep in the retry attempt got interrupted", e); + } + } + retryCount++; + } + String errorMessage = String.format("Exceeded max retry attempts. Failed to execute the Rest API call %s", uri); + log.error(errorMessage); + throw new RuntimeException(errorMessage); + } + + @VisibleForTesting + public void setSleepTimeMultiplier(int multiplier) { + sleepTimeMultiplier = multiplier; + } +} diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/BasicAuthInterceptor.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/BasicAuthInterceptor.java new file mode 100644 index 0000000000..ec37d6efd3 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/BasicAuthInterceptor.java @@ -0,0 +1,42 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.atlassian.rest; + +import org.opensearch.dataprepper.plugins.source.atlassian.configuration.AtlassianSourceConfig; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpRequest; +import org.springframework.http.client.ClientHttpRequestExecution; +import org.springframework.http.client.ClientHttpRequestInterceptor; +import org.springframework.http.client.ClientHttpResponse; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Base64; + + +public class BasicAuthInterceptor implements ClientHttpRequestInterceptor { + private final String username; + private final String password; + + public BasicAuthInterceptor(AtlassianSourceConfig config) { + this.username = config.getAuthenticationConfig().getBasicConfig().getUsername(); + this.password = config.getAuthenticationConfig().getBasicConfig().getPassword(); + } + + @Override + public ClientHttpResponse intercept(HttpRequest request, byte[] body, ClientHttpRequestExecution execution) throws IOException { + String auth = username + ":" + password; + byte[] encodedAuth = Base64.getEncoder().encode(auth.getBytes(StandardCharsets.US_ASCII)); + String authHeader = "Basic " + new String(encodedAuth); + request.getHeaders().set(HttpHeaders.AUTHORIZATION, authHeader); + return execution.execute(request, body); + } +} diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/CustomRestTemplateConfig.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/CustomRestTemplateConfig.java new file mode 100644 index 0000000000..eb78131f9f --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/CustomRestTemplateConfig.java @@ -0,0 +1,42 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.atlassian.rest; + + +import org.opensearch.dataprepper.plugins.source.atlassian.configuration.AtlassianSourceConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.rest.auth.AtlassianAuthConfig; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.http.client.ClientHttpRequestInterceptor; +import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; +import org.springframework.web.client.RestTemplate; + +import static org.opensearch.dataprepper.plugins.source.atlassian.utils.Constants.OAUTH2; + +@Configuration +public class CustomRestTemplateConfig { + + @Bean + public RestTemplate basicAuthRestTemplate(AtlassianSourceConfig config, AtlassianAuthConfig authConfig) { + RestTemplate restTemplate = new RestTemplate(); + restTemplate.setRequestFactory(new HttpComponentsClientHttpRequestFactory()); + ClientHttpRequestInterceptor httpInterceptor; + if (OAUTH2.equals(config.getAuthType())) { + httpInterceptor = new OAuth2RequestInterceptor(authConfig); + } else { + httpInterceptor = new BasicAuthInterceptor(config); + } + restTemplate.getInterceptors().add(httpInterceptor); + return restTemplate; + } + + +} diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/OAuth2RequestInterceptor.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/OAuth2RequestInterceptor.java new file mode 100644 index 0000000000..2eabfc87bd --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/OAuth2RequestInterceptor.java @@ -0,0 +1,36 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.atlassian.rest; + +import org.opensearch.dataprepper.plugins.source.atlassian.rest.auth.AtlassianAuthConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.rest.auth.AtlassianOauthConfig; +import org.springframework.http.HttpRequest; +import org.springframework.http.client.ClientHttpRequestExecution; +import org.springframework.http.client.ClientHttpRequestInterceptor; +import org.springframework.http.client.ClientHttpResponse; + +import java.io.IOException; + +public class OAuth2RequestInterceptor implements ClientHttpRequestInterceptor { + + private final AtlassianAuthConfig config; + + public OAuth2RequestInterceptor(AtlassianAuthConfig config) { + this.config = config; + } + + @Override + public ClientHttpResponse intercept(HttpRequest request, byte[] body, ClientHttpRequestExecution execution) throws IOException { + request.getHeaders().setBearerAuth(((AtlassianOauthConfig) config).getAccessToken()); + return execution.execute(request, body); + } + +} diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianAuthConfig.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianAuthConfig.java new file mode 100644 index 0000000000..60b9529339 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianAuthConfig.java @@ -0,0 +1,34 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.atlassian.rest.auth; + +/** + * The interface that defines the behaviour for Jira auth configs. + */ +public interface AtlassianAuthConfig { + + /** + * Returns the URL for the Jira instance. + * + * @return the URL for the Jira instance. + */ + String getUrl(); + + /** + * Initializes the credentials for the Jira instance. + */ + void initCredentials(); + + /** + * Renews the credentials for the Jira instance. + */ + void renewCredentials(); +} diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianAuthFactory.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianAuthFactory.java new file mode 100644 index 0000000000..690f7663b2 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianAuthFactory.java @@ -0,0 +1,41 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.atlassian.rest.auth; + +import org.opensearch.dataprepper.plugins.source.atlassian.configuration.AtlassianSourceConfig; +import org.springframework.beans.factory.FactoryBean; +import org.springframework.context.annotation.Configuration; + +import static org.opensearch.dataprepper.plugins.source.atlassian.utils.Constants.OAUTH2; + +@Configuration +public class AtlassianAuthFactory implements FactoryBean { + + private final AtlassianSourceConfig sourceConfig; + + public AtlassianAuthFactory(AtlassianSourceConfig sourceConfig) { + this.sourceConfig = sourceConfig; + } + + @Override + public AtlassianAuthConfig getObject() { + String authType = sourceConfig.getAuthType(); + if (OAUTH2.equals(authType)) { + return new AtlassianOauthConfig(sourceConfig); + } + return new AtlassianBasicAuthConfig(sourceConfig); + } + + @Override + public Class getObjectType() { + return AtlassianAuthConfig.class; + } +} diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianBasicAuthConfig.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianBasicAuthConfig.java new file mode 100644 index 0000000000..58819d3d72 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianBasicAuthConfig.java @@ -0,0 +1,45 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.atlassian.rest.auth; + + +import org.opensearch.dataprepper.plugins.source.atlassian.configuration.AtlassianSourceConfig; + +public class AtlassianBasicAuthConfig implements AtlassianAuthConfig { + + private String accountUrl; + private final AtlassianSourceConfig confluenceSourceConfig; + + public AtlassianBasicAuthConfig(AtlassianSourceConfig confluenceSourceConfig) { + this.confluenceSourceConfig = confluenceSourceConfig; + accountUrl = confluenceSourceConfig.getAccountUrl(); + if (!accountUrl.endsWith("/")) { + accountUrl += "/"; + } + } + + @Override + public String getUrl() { + return accountUrl; + } + + @Override + public void initCredentials() { + //do nothing for basic authentication + } + + @Override + public void renewCredentials() { + //do nothing for basic authentication + } + + +} diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianOauthConfig.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianOauthConfig.java new file mode 100644 index 0000000000..10c11a3c7b --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianOauthConfig.java @@ -0,0 +1,183 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.atlassian.rest.auth; + +import lombok.Getter; +import org.opensearch.dataprepper.plugins.source.atlassian.configuration.AtlassianSourceConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.configuration.Oauth2Config; +import org.opensearch.dataprepper.plugins.source.source_crawler.exception.UnAuthorizedException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.util.StringUtils; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.RestTemplate; + +import java.time.Instant; +import java.util.List; +import java.util.Map; + +import static org.opensearch.dataprepper.plugins.source.atlassian.utils.Constants.RETRY_ATTEMPT; +import static org.opensearch.dataprepper.plugins.source.atlassian.utils.Constants.SLASH; + +/** + * The type Jira service. + */ + +public class AtlassianOauthConfig implements AtlassianAuthConfig { + + public static final String OAuth2_URL = "https://api.atlassian.com/ex/jira/"; + public static final String ACCESSIBLE_RESOURCES = "https://api.atlassian.com/oauth/token/accessible-resources"; + public static final String TOKEN_LOCATION = "https://auth.atlassian.com/oauth/token"; + + public static final String EXPIRES_IN = "expires_in"; + public static final String REFRESH_TOKEN = "refresh_token"; + public static final String ACCESS_TOKEN = "access_token"; + private static final Logger log = LoggerFactory.getLogger(AtlassianOauthConfig.class); + RestTemplate restTemplate = new RestTemplate(); + private String url; + @Getter + private int expiresInSeconds = 0; + @Getter + private Instant expireTime = Instant.ofEpochMilli(0); + @Getter + private String accessToken; + @Getter + private String refreshToken; + private String cloudId = null; + private final String clientId; + private final String clientSecret; + private final AtlassianSourceConfig confluenceSourceConfig; + private final Object cloudIdFetchLock = new Object(); + private final Object tokenRenewLock = new Object(); + + public AtlassianOauthConfig(AtlassianSourceConfig confluenceSourceConfig) { + this.confluenceSourceConfig = confluenceSourceConfig; + this.accessToken = (String) confluenceSourceConfig.getAuthenticationConfig().getOauth2Config() + .getAccessToken().getValue(); + this.refreshToken = (String) confluenceSourceConfig.getAuthenticationConfig() + .getOauth2Config().getRefreshToken().getValue(); + this.clientId = confluenceSourceConfig.getAuthenticationConfig().getOauth2Config().getClientId(); + this.clientSecret = confluenceSourceConfig.getAuthenticationConfig().getOauth2Config().getClientSecret(); + } + + public String getJiraAccountCloudId() { + log.info("Getting Jira Account Cloud ID"); + synchronized (cloudIdFetchLock) { + if (this.cloudId != null) { + //Someone else must have initialized it + return this.cloudId; + } + + int retryCount = 0; + while (retryCount < RETRY_ATTEMPT) { + retryCount++; + try { + HttpHeaders headers = new HttpHeaders(); + headers.setBearerAuth(accessToken); + headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED); + HttpEntity entity = new HttpEntity<>(headers); + ResponseEntity exchangeResponse = + restTemplate.exchange(ACCESSIBLE_RESOURCES, HttpMethod.GET, entity, Object.class); + List> listResponse = (List>) exchangeResponse.getBody(); + Map response = listResponse.get(0); + this.cloudId = (String) response.get("id"); + return this.cloudId; + } catch (HttpClientErrorException e) { + if (e.getRawStatusCode() == HttpStatus.UNAUTHORIZED.value()) { + renewCredentials(); + } + log.error("Error occurred while accessing resources: ", e); + } + } + throw new UnAuthorizedException(String.format("Access token expired. Unable to renew even after %s attempts", RETRY_ATTEMPT)); + } + } + + public void renewCredentials() { + Instant currentTime = Instant.now(); + if (expireTime.isAfter(currentTime)) { + //There is still time to renew or someone else must have already renewed it + return; + } + + synchronized (tokenRenewLock) { + if (expireTime.isAfter(currentTime)) { + //Someone else must have already renewed it + return; + } + + log.info("Renewing access token and refresh token pair for Jira Connector."); + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + String payloadTemplate = "{\"grant_type\": \"%s\", \"client_id\": \"%s\", \"client_secret\": \"%s\", \"refresh_token\": \"%s\"}"; + String payload = String.format(payloadTemplate, "refresh_token", clientId, clientSecret, refreshToken); + HttpEntity entity = new HttpEntity<>(payload, headers); + + Oauth2Config oauth2Config = confluenceSourceConfig.getAuthenticationConfig().getOauth2Config(); + try { + ResponseEntity responseEntity = restTemplate.postForEntity(TOKEN_LOCATION, entity, Map.class); + Map oauthClientResponse = responseEntity.getBody(); + this.accessToken = (String) oauthClientResponse.get(ACCESS_TOKEN); + this.refreshToken = (String) oauthClientResponse.get(REFRESH_TOKEN); + this.expiresInSeconds = (int) oauthClientResponse.get(EXPIRES_IN); + this.expireTime = Instant.now().plusSeconds(expiresInSeconds); + // updating config object's PluginConfigVariable so that it updates the underlying Secret store + oauth2Config.getAccessToken().setValue(this.accessToken); + oauth2Config.getRefreshToken().setValue(this.refreshToken); + log.info("Access Token and Refresh Token pair is now refreshed. Corresponding Secret store key updated."); + } catch (HttpClientErrorException ex) { + this.expireTime = Instant.ofEpochMilli(0); + this.expiresInSeconds = 0; + HttpStatus statusCode = ex.getStatusCode(); + log.error("Failed to renew access token. Status code: {}, Error Message: {}", + statusCode, ex.getMessage()); + if (statusCode == HttpStatus.FORBIDDEN || statusCode == HttpStatus.UNAUTHORIZED) { + log.info("Trying to refresh the secrets"); + // Refreshing the secrets. It should help if someone already renewed the tokens. + // Refreshing one of the secret refreshes the entire store so triggering refresh on just one + oauth2Config.getAccessToken().refresh(); + this.accessToken = (String) oauth2Config.getAccessToken().getValue(); + this.refreshToken = (String) oauth2Config.getRefreshToken().getValue(); + this.expireTime = Instant.now().plusSeconds(10); + } + throw new RuntimeException("Failed to renew access token message:" + ex.getMessage(), ex); + } + } + } + + @Override + public String getUrl() { + if (!StringUtils.hasLength(url)) { + synchronized (cloudIdFetchLock) { + if (!StringUtils.hasLength(url)) { + initCredentials(); + } + } + } + return url; + } + + /** + * Method for getting Jira url based on auth type. + */ + @Override + public void initCredentials() { + //For OAuth based flow, we use a different Jira url + this.cloudId = getJiraAccountCloudId(); + this.url = OAuth2_URL + this.cloudId + SLASH; + } +} diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/utils/Constants.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/utils/Constants.java new file mode 100644 index 0000000000..e4065d2c3f --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/utils/Constants.java @@ -0,0 +1,23 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.atlassian.utils; + +/** + * The type Constants. + */ +public class Constants { + + public static final int RETRY_ATTEMPT = 6; + + public static final String OAUTH2 = "OAuth2"; + public static final String BASIC = "Basic"; + public static final String SLASH = "/"; +} \ No newline at end of file diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java index 877960c634..bc72bf3496 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java @@ -13,11 +13,11 @@ import io.micrometer.core.instrument.Counter; import lombok.extern.slf4j.Slf4j; import org.opensearch.dataprepper.metrics.PluginMetrics; -import org.opensearch.dataprepper.plugins.source.confluence.exception.BadRequestException; import org.opensearch.dataprepper.plugins.source.confluence.models.ConfluenceItem; import org.opensearch.dataprepper.plugins.source.confluence.models.ConfluenceSearchResults; import org.opensearch.dataprepper.plugins.source.confluence.rest.ConfluenceRestClient; import org.opensearch.dataprepper.plugins.source.confluence.utils.ConfluenceConfigHelper; +import org.opensearch.dataprepper.plugins.source.source_crawler.exception.BadRequestException; import org.opensearch.dataprepper.plugins.source.source_crawler.model.ItemInfo; import org.springframework.util.CollectionUtils; diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClient.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClient.java index f0bf79be14..45087d153f 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClient.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClient.java @@ -16,11 +16,11 @@ import io.micrometer.core.instrument.Timer; import lombok.extern.slf4j.Slf4j; import org.opensearch.dataprepper.metrics.PluginMetrics; -import org.opensearch.dataprepper.plugins.source.confluence.exception.BadRequestException; -import org.opensearch.dataprepper.plugins.source.confluence.exception.UnAuthorizedException; import org.opensearch.dataprepper.plugins.source.confluence.models.ConfluenceSearchResults; import org.opensearch.dataprepper.plugins.source.confluence.rest.auth.ConfluenceAuthConfig; -import org.opensearch.dataprepper.plugins.source.confluence.utils.AddressValidation; +import org.opensearch.dataprepper.plugins.source.source_crawler.exception.BadRequestException; +import org.opensearch.dataprepper.plugins.source.source_crawler.exception.UnAuthorizedException; +import org.opensearch.dataprepper.plugins.source.source_crawler.utils.AddressValidation; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.client.HttpClientErrorException; diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceOauthConfig.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceOauthConfig.java index dea42f2917..2c1fb08afb 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceOauthConfig.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceOauthConfig.java @@ -13,7 +13,7 @@ import lombok.Getter; import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceSourceConfig; import org.opensearch.dataprepper.plugins.source.confluence.configuration.Oauth2Config; -import org.opensearch.dataprepper.plugins.source.confluence.exception.UnAuthorizedException; +import org.opensearch.dataprepper.plugins.source.source_crawler.exception.UnAuthorizedException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.HttpEntity; diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/ConfluenceConfigHelper.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/ConfluenceConfigHelper.java index 9f36bfe91d..39eecc8417 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/ConfluenceConfigHelper.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/ConfluenceConfigHelper.java @@ -13,6 +13,7 @@ import lombok.extern.slf4j.Slf4j; import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceSourceConfig; +import org.opensearch.dataprepper.plugins.source.source_crawler.utils.AddressValidation; import java.util.ArrayList; import java.util.List; diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/exception/BadRequestException.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/exception/BadRequestException.java similarity index 89% rename from data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/exception/BadRequestException.java rename to data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/exception/BadRequestException.java index 30a4dd36e2..4cbe181011 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/exception/BadRequestException.java +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/exception/BadRequestException.java @@ -8,7 +8,7 @@ * */ -package org.opensearch.dataprepper.plugins.source.confluence.exception; +package org.opensearch.dataprepper.plugins.source.source_crawler.exception; /** * Exception to indicate a bad REST call has been made. diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/exception/UnAuthorizedException.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/exception/UnAuthorizedException.java similarity index 90% rename from data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/exception/UnAuthorizedException.java rename to data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/exception/UnAuthorizedException.java index 48095d3e45..ed005cd90e 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/exception/UnAuthorizedException.java +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/exception/UnAuthorizedException.java @@ -8,7 +8,7 @@ * */ -package org.opensearch.dataprepper.plugins.source.confluence.exception; +package org.opensearch.dataprepper.plugins.source.source_crawler.exception; /** * Exception to indicate unauthorized access. diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/AddressValidation.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/AddressValidation.java similarity index 91% rename from data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/AddressValidation.java rename to data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/AddressValidation.java index 126ac3ec95..b549fb2a5e 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/AddressValidation.java +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/AddressValidation.java @@ -8,11 +8,11 @@ * */ -package org.opensearch.dataprepper.plugins.source.confluence.utils; +package org.opensearch.dataprepper.plugins.source.source_crawler.utils; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; -import org.opensearch.dataprepper.plugins.source.confluence.exception.BadRequestException; +import org.opensearch.dataprepper.plugins.source.source_crawler.exception.BadRequestException; import java.net.InetAddress; import java.net.MalformedURLException; diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/exception/BadRequestExceptionTest.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/exception/BadRequestExceptionTest.java similarity index 91% rename from data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/exception/BadRequestExceptionTest.java rename to data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/exception/BadRequestExceptionTest.java index 4a38014021..249d0260b5 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/exception/BadRequestExceptionTest.java +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/exception/BadRequestExceptionTest.java @@ -8,12 +8,11 @@ * */ -package org.opensearch.dataprepper.plugins.source.confluence.exception; +package org.opensearch.dataprepper.plugins.source.source_crawler.exception; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; -import org.opensearch.dataprepper.plugins.source.confluence.exception.BadRequestException; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/exception/UnAuthorizedExceptionTest.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/exception/UnAuthorizedExceptionTest.java similarity index 91% rename from data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/exception/UnAuthorizedExceptionTest.java rename to data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/exception/UnAuthorizedExceptionTest.java index a8a979d68f..ecebc1aad2 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/exception/UnAuthorizedExceptionTest.java +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/exception/UnAuthorizedExceptionTest.java @@ -8,12 +8,11 @@ * */ -package org.opensearch.dataprepper.plugins.source.confluence.exception; +package org.opensearch.dataprepper.plugins.source.source_crawler.exception; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; -import org.opensearch.dataprepper.plugins.source.confluence.exception.UnAuthorizedException; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/utils/AddressValidationTest.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/AddressValidationTest.java similarity index 93% rename from data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/utils/AddressValidationTest.java rename to data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/AddressValidationTest.java index 862d01878e..c3e3e7a0e2 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/utils/AddressValidationTest.java +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/AddressValidationTest.java @@ -8,10 +8,10 @@ * */ -package org.opensearch.dataprepper.plugins.source.confluence.utils; +package org.opensearch.dataprepper.plugins.source.source_crawler.utils; import org.junit.jupiter.api.Test; -import org.opensearch.dataprepper.plugins.source.confluence.exception.BadRequestException; +import org.opensearch.dataprepper.plugins.source.source_crawler.exception.BadRequestException; import java.net.InetAddress; import java.net.MalformedURLException; @@ -27,8 +27,6 @@ public class AddressValidationTest { @Test void testInstanceCreation() { assertNotNull(new AddressValidation()); - assertNotNull(new Constants()); - assertNotNull(new CqlConstants()); } @Test diff --git a/settings.gradle b/settings.gradle index bee7e45f28..39602176ef 100644 --- a/settings.gradle +++ b/settings.gradle @@ -190,4 +190,5 @@ include 'data-prepper-plugins:saas-source-plugins' include 'data-prepper-plugins:saas-source-plugins:source-crawler' include 'data-prepper-plugins:saas-source-plugins:jira-source' include 'data-prepper-plugins:saas-source-plugins:confluence-source' +include 'data-prepper-plugins:saas-source-plugins:atlassian-commons' From 0e89bf00c233d7195aa3fa4e10cb37da11370b3a Mon Sep 17 00:00:00 2001 From: Santhosh Gandhe <1909520+san81@users.noreply.github.com> Date: Mon, 3 Feb 2025 15:06:06 -0800 Subject: [PATCH 07/14] functional with making use of Atlassian Commons package Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com> --- .../AtlassianSourceConfig.java | 3 +- .../atlassian/rest/BasicAuthInterceptor.java | 2 +- .../rest/CustomRestTemplateConfig.java | 2 +- .../rest/auth/AtlassianAuthFactory.java | 2 +- .../rest/auth/AtlassianBasicAuthConfig.java | 2 +- .../rest/auth/AtlassianOauthConfig.java | 2 +- .../confluence-source/build.gradle | 1 + .../source/confluence/ConfluenceItemInfo.java | 2 +- .../source/confluence/ConfluenceService.java | 52 ++--- .../source/confluence/ConfluenceSource.java | 17 +- .../confluence/ConfluenceSourceConfig.java | 37 +--- .../configuration/AuthenticationConfig.java | 45 ----- .../confluence/configuration/BasicConfig.java | 29 --- .../configuration/Oauth2Config.java | 36 ---- .../confluence/rest/BasicAuthInterceptor.java | 42 ---- .../confluence/rest/ConfluenceRestClient.java | 50 +---- .../rest/CustomRestTemplateConfig.java | 42 ---- .../rest/OAuth2RequestInterceptor.java | 36 ---- .../rest/auth/ConfluenceAuthConfig.java | 34 ---- .../rest/auth/ConfluenceAuthFactory.java | 41 ---- .../rest/auth/ConfluenceBasicAuthConfig.java | 45 ----- .../rest/auth/ConfluenceOauthConfig.java | 183 ------------------ .../source/confluence/utils/Constants.java | 1 - .../rest/ConfluenceRestClientTest.java | 12 +- 24 files changed, 58 insertions(+), 660 deletions(-) rename data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/{configuration => }/AtlassianSourceConfig.java (89%) delete mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/AuthenticationConfig.java delete mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/BasicConfig.java delete mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/Oauth2Config.java delete mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/BasicAuthInterceptor.java delete mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/CustomRestTemplateConfig.java delete mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/OAuth2RequestInterceptor.java delete mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceAuthConfig.java delete mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceAuthFactory.java delete mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceBasicAuthConfig.java delete mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceOauthConfig.java diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/configuration/AtlassianSourceConfig.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/AtlassianSourceConfig.java similarity index 89% rename from data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/configuration/AtlassianSourceConfig.java rename to data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/AtlassianSourceConfig.java index 448206469a..bc661f96af 100644 --- a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/configuration/AtlassianSourceConfig.java +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/AtlassianSourceConfig.java @@ -8,11 +8,12 @@ * */ -package org.opensearch.dataprepper.plugins.source.atlassian.configuration; +package org.opensearch.dataprepper.plugins.source.atlassian; import com.fasterxml.jackson.annotation.JsonProperty; import jakarta.validation.Valid; import lombok.Getter; +import org.opensearch.dataprepper.plugins.source.atlassian.configuration.AuthenticationConfig; import org.opensearch.dataprepper.plugins.source.source_crawler.base.CrawlerSourceConfig; import java.util.List; diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/BasicAuthInterceptor.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/BasicAuthInterceptor.java index ec37d6efd3..80c5e9729b 100644 --- a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/BasicAuthInterceptor.java +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/BasicAuthInterceptor.java @@ -10,7 +10,7 @@ package org.opensearch.dataprepper.plugins.source.atlassian.rest; -import org.opensearch.dataprepper.plugins.source.atlassian.configuration.AtlassianSourceConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpRequest; import org.springframework.http.client.ClientHttpRequestExecution; diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/CustomRestTemplateConfig.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/CustomRestTemplateConfig.java index eb78131f9f..a8cd1c131d 100644 --- a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/CustomRestTemplateConfig.java +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/CustomRestTemplateConfig.java @@ -11,7 +11,7 @@ package org.opensearch.dataprepper.plugins.source.atlassian.rest; -import org.opensearch.dataprepper.plugins.source.atlassian.configuration.AtlassianSourceConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig; import org.opensearch.dataprepper.plugins.source.atlassian.rest.auth.AtlassianAuthConfig; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianAuthFactory.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianAuthFactory.java index 690f7663b2..e10f2e3d2a 100644 --- a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianAuthFactory.java +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianAuthFactory.java @@ -10,7 +10,7 @@ package org.opensearch.dataprepper.plugins.source.atlassian.rest.auth; -import org.opensearch.dataprepper.plugins.source.atlassian.configuration.AtlassianSourceConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig; import org.springframework.beans.factory.FactoryBean; import org.springframework.context.annotation.Configuration; diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianBasicAuthConfig.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianBasicAuthConfig.java index 58819d3d72..68d81bd887 100644 --- a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianBasicAuthConfig.java +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianBasicAuthConfig.java @@ -11,7 +11,7 @@ package org.opensearch.dataprepper.plugins.source.atlassian.rest.auth; -import org.opensearch.dataprepper.plugins.source.atlassian.configuration.AtlassianSourceConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig; public class AtlassianBasicAuthConfig implements AtlassianAuthConfig { diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianOauthConfig.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianOauthConfig.java index 10c11a3c7b..654785c684 100644 --- a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianOauthConfig.java +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianOauthConfig.java @@ -11,7 +11,7 @@ package org.opensearch.dataprepper.plugins.source.atlassian.rest.auth; import lombok.Getter; -import org.opensearch.dataprepper.plugins.source.atlassian.configuration.AtlassianSourceConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig; import org.opensearch.dataprepper.plugins.source.atlassian.configuration.Oauth2Config; import org.opensearch.dataprepper.plugins.source.source_crawler.exception.UnAuthorizedException; import org.slf4j.Logger; diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/build.gradle b/data-prepper-plugins/saas-source-plugins/confluence-source/build.gradle index b59ab8d251..07308a50f7 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/build.gradle +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/build.gradle @@ -5,6 +5,7 @@ plugins { dependencies { implementation project(path: ':data-prepper-plugins:saas-source-plugins:source-crawler') + implementation project(path: ':data-prepper-plugins:saas-source-plugins:atlassian-commons') implementation project(path: ':data-prepper-api') implementation project(path: ':data-prepper-plugins:aws-plugin-api') implementation project(path: ':data-prepper-plugins:buffer-common') diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceItemInfo.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceItemInfo.java index c6048caf65..f017f2588d 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceItemInfo.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceItemInfo.java @@ -137,7 +137,7 @@ public ConfluenceItemInfoBuilder withSpace(String space) { return this; } - public ConfluenceItemInfoBuilder withIssueBean(ConfluenceItem contentItem) { + public ConfluenceItemInfoBuilder withContentBean(ConfluenceItem contentItem) { Map contentItemMetadata = new HashMap<>(); contentItemMetadata.put(SPACE_KEY, contentItem.getSpaceItem().getKey()); contentItemMetadata.put(CONTENT_TITLE, contentItem.getTitle()); diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java index bc72bf3496..5893e7065d 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java @@ -48,7 +48,7 @@ /** - * Service class for interactive external Atlassian jira SaaS service and fetch required details using their rest apis. + * Service class for interactive external Atlassian Confluence SaaS service and fetch required details using their rest apis. */ @Slf4j @@ -73,7 +73,7 @@ public ConfluenceService(ConfluenceSourceConfig confluenceSourceConfig, } /** - * Get jira entities. + * Get Confluence entities. * * @param configuration the configuration. * @param timestamp timestamp. @@ -89,7 +89,7 @@ public String getContent(String contentId) { } /** - * Method for building Issue Item Info. + * Method for building Content Item Info. * * @param configuration Input Parameter * @param timestamp Input Parameter @@ -101,11 +101,11 @@ private void searchForNewContentAndAddToQueue(ConfluenceSourceConfig configurati int total; int startAt = 0; do { - ConfluenceSearchResults searchIssues = confluenceRestClient.getAllContent(cql, startAt); - List issueList = new ArrayList<>(searchIssues.getResults()); - total = searchIssues.getSize(); - startAt += searchIssues.getResults().size(); - addItemsToQueue(issueList, itemInfoQueue); + ConfluenceSearchResults searchContentItems = confluenceRestClient.getAllContent(cql, startAt); + List contentList = new ArrayList<>(searchContentItems.getResults()); + total = searchContentItems.getSize(); + startAt += searchContentItems.getResults().size(); + addItemsToQueue(contentList, itemInfoQueue); } while (startAt < total); searchResultsFoundCounter.increment(total); log.info("Number of tickets found in search api call: {}", total); @@ -114,12 +114,12 @@ private void searchForNewContentAndAddToQueue(ConfluenceSourceConfig configurati /** * Add items to queue. * - * @param issueList Issue list. + * @param contentList Content list. * @param itemInfoQueue Item info queue. */ - private void addItemsToQueue(List issueList, Queue itemInfoQueue) { - issueList.forEach(issue -> itemInfoQueue.add(ConfluenceItemInfo.builder() - .withEventTime(Instant.now()).withIssueBean(issue).build())); + private void addItemsToQueue(List contentList, Queue itemInfoQueue) { + contentList.forEach(contentItem -> itemInfoQueue.add(ConfluenceItemInfo.builder() + .withEventTime(Instant.now()).withContentBean(contentItem).build())); } @@ -160,7 +160,7 @@ private StringBuilder createContentFilterCriteria(ConfluenceSourceConfig configu .append(CLOSING_ROUND_BRACKET); } - log.error("Created issue filter criteria JiraQl query: {}", cQl); + log.info("Created content filter criteria ConfluenceQl query: {}", cQl); return cQl; } @@ -170,25 +170,25 @@ private StringBuilder createContentFilterCriteria(ConfluenceSourceConfig configu * @param configuration Input Parameter */ private void validateSpaceFilters(ConfluenceSourceConfig configuration) { - log.trace("Validating project filters"); + log.trace("Validating space filters"); List badFilters = new ArrayList<>(); - Set includedProjects = new HashSet<>(); + Set includedSpaces = new HashSet<>(); List includedAndExcludedSpaces = new ArrayList<>(); Pattern regex = Pattern.compile("[^A-Z0-9]"); - ConfluenceConfigHelper.getSpacesNameIncludeFilter(configuration).forEach(projectFilter -> { - Matcher matcher = regex.matcher(projectFilter); - includedProjects.add(projectFilter); - if (matcher.find() || projectFilter.length() <= 1 || projectFilter.length() > 10) { - badFilters.add(projectFilter); + ConfluenceConfigHelper.getSpacesNameIncludeFilter(configuration).forEach(spaceFilter -> { + Matcher matcher = regex.matcher(spaceFilter); + includedSpaces.add(spaceFilter); + if (matcher.find() || spaceFilter.length() <= 1 || spaceFilter.length() > 10) { + badFilters.add(spaceFilter); } }); - ConfluenceConfigHelper.getSpacesNameExcludeFilter(configuration).forEach(projectFilter -> { - Matcher matcher = regex.matcher(projectFilter); - if (includedProjects.contains(projectFilter)) { - includedAndExcludedSpaces.add(projectFilter); + ConfluenceConfigHelper.getSpacesNameExcludeFilter(configuration).forEach(spaceFilter -> { + Matcher matcher = regex.matcher(spaceFilter); + if (includedSpaces.contains(spaceFilter)) { + includedAndExcludedSpaces.add(spaceFilter); } - if (matcher.find() || projectFilter.length() <= 1 || projectFilter.length() > 10) { - badFilters.add(projectFilter); + if (matcher.find() || spaceFilter.length() <= 1 || spaceFilter.length() > 10) { + badFilters.add(spaceFilter); } }); if (!badFilters.isEmpty()) { diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSource.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSource.java index 69294be228..d126b0da10 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSource.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSource.java @@ -20,7 +20,8 @@ import org.opensearch.dataprepper.model.plugin.PluginFactory; import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.model.source.Source; -import org.opensearch.dataprepper.plugins.source.confluence.rest.auth.ConfluenceAuthConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.rest.auth.AtlassianAuthConfig; import org.opensearch.dataprepper.plugins.source.confluence.utils.ConfluenceConfigHelper; import org.opensearch.dataprepper.plugins.source.source_crawler.CrawlerApplicationContextMarker; import org.opensearch.dataprepper.plugins.source.source_crawler.base.Crawler; @@ -33,37 +34,37 @@ /** - * JiraConnector connector entry point. + * Confluence connector entry point. */ @DataPrepperPlugin(name = PLUGIN_NAME, pluginType = Source.class, pluginConfigurationType = ConfluenceSourceConfig.class, - packagesToScan = {CrawlerApplicationContextMarker.class, ConfluenceSource.class} + packagesToScan = {CrawlerApplicationContextMarker.class, AtlassianSourceConfig.class, ConfluenceSource.class} ) public class ConfluenceSource extends CrawlerSourcePlugin { private static final Logger log = LoggerFactory.getLogger(ConfluenceSource.class); private final ConfluenceSourceConfig confluenceSourceConfig; - private final ConfluenceAuthConfig jiraOauthConfig; + private final AtlassianAuthConfig jiraOauthConfig; @DataPrepperPluginConstructor public ConfluenceSource(final PluginMetrics pluginMetrics, final ConfluenceSourceConfig confluenceSourceConfig, - final ConfluenceAuthConfig jiraOauthConfig, + final AtlassianAuthConfig jiraOauthConfig, final PluginFactory pluginFactory, final AcknowledgementSetManager acknowledgementSetManager, Crawler crawler, PluginExecutorServiceProvider executorServiceProvider) { super(PLUGIN_NAME, pluginMetrics, confluenceSourceConfig, pluginFactory, acknowledgementSetManager, crawler, executorServiceProvider); - log.info("Creating Jira Source Plugin"); + log.info("Creating Confluence Source Plugin"); this.confluenceSourceConfig = confluenceSourceConfig; this.jiraOauthConfig = jiraOauthConfig; } @Override public void start(Buffer> buffer) { - log.info("Starting Jira Source Plugin... "); + log.info("Starting Confluence Source Plugin... "); ConfluenceConfigHelper.validateConfig(confluenceSourceConfig); jiraOauthConfig.initCredentials(); super.start(buffer); @@ -71,7 +72,7 @@ public void start(Buffer> buffer) { @Override public void stop() { - log.info("Stopping Jira Source Plugin"); + log.info("Stopping Confluence Source Plugin"); super.stop(); } diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceConfig.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceConfig.java index 57afe256d2..b22ad46721 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceConfig.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceConfig.java @@ -11,39 +11,13 @@ package org.opensearch.dataprepper.plugins.source.confluence; import com.fasterxml.jackson.annotation.JsonProperty; -import jakarta.validation.Valid; import lombok.Getter; -import org.opensearch.dataprepper.plugins.source.confluence.configuration.AuthenticationConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig; import org.opensearch.dataprepper.plugins.source.confluence.configuration.FilterConfig; import org.opensearch.dataprepper.plugins.source.source_crawler.base.CrawlerSourceConfig; -import java.util.List; - @Getter -public class ConfluenceSourceConfig implements CrawlerSourceConfig { - - private static final int DEFAULT_BATCH_SIZE = 50; - - /** - * Jira account url - */ - @JsonProperty("hosts") - private List hosts; - - /** - * Authentication Config to Access Jira - */ - @JsonProperty("authentication") - @Valid - private AuthenticationConfig authenticationConfig; - - /** - * Batch size for fetching tickets - */ - @JsonProperty("batch_size") - private int batchSize = DEFAULT_BATCH_SIZE; - - +public class ConfluenceSourceConfig extends AtlassianSourceConfig implements CrawlerSourceConfig { /** * Filter Config to filter what tickets get ingested */ @@ -57,11 +31,4 @@ public class ConfluenceSourceConfig implements CrawlerSourceConfig { @JsonProperty("acknowledgments") private boolean acknowledgments = false; - public String getAccountUrl() { - return this.getHosts().get(0); - } - - public String getAuthType() { - return this.getAuthenticationConfig().getAuthType(); - } } diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/AuthenticationConfig.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/AuthenticationConfig.java deleted file mode 100644 index d5632f5f2e..0000000000 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/AuthenticationConfig.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - */ - -package org.opensearch.dataprepper.plugins.source.confluence.configuration; - -import com.fasterxml.jackson.annotation.JsonProperty; -import jakarta.validation.Valid; -import jakarta.validation.constraints.AssertTrue; -import lombok.Getter; - -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.BASIC; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.OAUTH2; - -@Getter -public class AuthenticationConfig { - @JsonProperty("basic") - @Valid - private BasicConfig basicConfig; - - @JsonProperty("oauth2") - @Valid - private Oauth2Config oauth2Config; - - @AssertTrue(message = "Authentication config should have either basic or oauth2") - private boolean isValidAuthenticationConfig() { - boolean hasBasic = basicConfig != null; - boolean hasOauth = oauth2Config != null; - return hasBasic ^ hasOauth; - } - - public String getAuthType() { - if (basicConfig != null) { - return BASIC; - } else { - return OAUTH2; - } - } -} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/BasicConfig.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/BasicConfig.java deleted file mode 100644 index 1cbb6de933..0000000000 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/BasicConfig.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - */ - -package org.opensearch.dataprepper.plugins.source.confluence.configuration; - -import com.fasterxml.jackson.annotation.JsonProperty; -import jakarta.validation.constraints.AssertTrue; -import lombok.Getter; - -@Getter -public class BasicConfig { - @JsonProperty("username") - private String username; - - @JsonProperty("password") - private String password; - - @AssertTrue(message = "Username and Password are both required for Basic Auth") - private boolean isBasicConfigValid() { - return username != null && password != null; - } -} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/Oauth2Config.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/Oauth2Config.java deleted file mode 100644 index 22daa7841b..0000000000 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/Oauth2Config.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - */ - -package org.opensearch.dataprepper.plugins.source.confluence.configuration; - -import com.fasterxml.jackson.annotation.JsonProperty; -import jakarta.validation.constraints.AssertTrue; -import lombok.Getter; -import org.opensearch.dataprepper.model.plugin.PluginConfigVariable; - -@Getter -public class Oauth2Config { - @JsonProperty("client_id") - private String clientId; - - @JsonProperty("client_secret") - private String clientSecret; - - @JsonProperty("access_token") - private PluginConfigVariable accessToken; - - @JsonProperty("refresh_token") - private PluginConfigVariable refreshToken; - - @AssertTrue(message = "Client ID, Client Secret, Access Token, and Refresh Token are both required for Oauth2") - private boolean isOauth2ConfigValid() { - return clientId != null && clientSecret != null && accessToken != null && refreshToken != null; - } -} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/BasicAuthInterceptor.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/BasicAuthInterceptor.java deleted file mode 100644 index 60bada4963..0000000000 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/BasicAuthInterceptor.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - */ - -package org.opensearch.dataprepper.plugins.source.confluence.rest; - -import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceSourceConfig; -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpRequest; -import org.springframework.http.client.ClientHttpRequestExecution; -import org.springframework.http.client.ClientHttpRequestInterceptor; -import org.springframework.http.client.ClientHttpResponse; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.Base64; - - -public class BasicAuthInterceptor implements ClientHttpRequestInterceptor { - private final String username; - private final String password; - - public BasicAuthInterceptor(ConfluenceSourceConfig config) { - this.username = config.getAuthenticationConfig().getBasicConfig().getUsername(); - this.password = config.getAuthenticationConfig().getBasicConfig().getPassword(); - } - - @Override - public ClientHttpResponse intercept(HttpRequest request, byte[] body, ClientHttpRequestExecution execution) throws IOException { - String auth = username + ":" + password; - byte[] encodedAuth = Base64.getEncoder().encode(auth.getBytes(StandardCharsets.US_ASCII)); - String authHeader = "Basic " + new String(encodedAuth); - request.getHeaders().set(HttpHeaders.AUTHORIZATION, authHeader); - return execution.execute(request, body); - } -} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClient.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClient.java index 45087d153f..bcda06d076 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClient.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClient.java @@ -16,30 +16,22 @@ import io.micrometer.core.instrument.Timer; import lombok.extern.slf4j.Slf4j; import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.plugins.source.atlassian.rest.AtlassianRestClient; +import org.opensearch.dataprepper.plugins.source.atlassian.rest.auth.AtlassianAuthConfig; import org.opensearch.dataprepper.plugins.source.confluence.models.ConfluenceSearchResults; -import org.opensearch.dataprepper.plugins.source.confluence.rest.auth.ConfluenceAuthConfig; -import org.opensearch.dataprepper.plugins.source.source_crawler.exception.BadRequestException; -import org.opensearch.dataprepper.plugins.source.source_crawler.exception.UnAuthorizedException; -import org.opensearch.dataprepper.plugins.source.source_crawler.utils.AddressValidation; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; -import org.springframework.web.client.HttpClientErrorException; import org.springframework.web.client.RestTemplate; import org.springframework.web.util.UriComponentsBuilder; import javax.inject.Named; import java.net.URI; -import java.util.List; -import static org.opensearch.dataprepper.logging.DataPrepperMarkers.NOISY; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.RETRY_ATTEMPT; import static org.opensearch.dataprepper.plugins.source.confluence.utils.CqlConstants.CQL_FIELD; import static org.opensearch.dataprepper.plugins.source.confluence.utils.CqlConstants.EXPAND_FIELD; import static org.opensearch.dataprepper.plugins.source.confluence.utils.CqlConstants.EXPAND_VALUE; @Slf4j @Named -public class ConfluenceRestClient { +public class ConfluenceRestClient extends AtlassianRestClient { public static final String REST_API_SEARCH = "wiki/rest/api/content/search"; public static final String REST_API_FETCH_CONTENT = "wiki/rest/api/content/"; @@ -48,21 +40,21 @@ public class ConfluenceRestClient { public static final String FIFTY = "50"; public static final String START_AT = "startAt"; public static final String MAX_RESULT = "maxResults"; - public static final List RETRY_ATTEMPT_SLEEP_TIME = List.of(1, 2, 5, 10, 20, 40); private static final String PAGE_FETCH_LATENCY_TIMER = "pageFetchLatency"; private static final String SEARCH_CALL_LATENCY_TIMER = "searchCallLatency"; private static final String SPACES_FETCH_LATENCY_TIMER = "spacesFetchLatency"; private static final String PAGES_REQUESTED = "pagesRequested"; private int sleepTimeMultiplier = 1000; private final RestTemplate restTemplate; - private final ConfluenceAuthConfig authConfig; + private final AtlassianAuthConfig authConfig; private final Timer contentFetchLatencyTimer; private final Timer searchCallLatencyTimer; private final Timer spaceFetchLatencyTimer; private final Counter contentRequestedCounter; - public ConfluenceRestClient(RestTemplate restTemplate, ConfluenceAuthConfig authConfig, + public ConfluenceRestClient(RestTemplate restTemplate, AtlassianAuthConfig authConfig, PluginMetrics pluginMetrics) { + super(restTemplate, authConfig); this.restTemplate = restTemplate; this.authConfig = authConfig; @@ -107,36 +99,6 @@ public String getContent(String contentId) { return invokeRestApi(uri, String.class).getBody(); } - private ResponseEntity invokeRestApi(URI uri, Class responseType) throws BadRequestException { - AddressValidation.validateInetAddress(AddressValidation.getInetAddress(uri.toString())); - int retryCount = 0; - while (retryCount < RETRY_ATTEMPT) { - try { - return restTemplate.getForEntity(uri, responseType); - } catch (HttpClientErrorException ex) { - HttpStatus statusCode = ex.getStatusCode(); - String statusMessage = ex.getMessage(); - log.error("An exception has occurred while getting response from Jira search API {}", ex.getMessage()); - if (statusCode == HttpStatus.FORBIDDEN) { - throw new UnAuthorizedException(statusMessage); - } else if (statusCode == HttpStatus.UNAUTHORIZED) { - log.error(NOISY, "Token expired. We will try to renew the tokens now", ex); - authConfig.renewCredentials(); - } else if (statusCode == HttpStatus.TOO_MANY_REQUESTS) { - log.error(NOISY, "Hitting API rate limit. Backing off with sleep timer.", ex); - } - try { - Thread.sleep((long) RETRY_ATTEMPT_SLEEP_TIME.get(retryCount) * sleepTimeMultiplier); - } catch (InterruptedException e) { - throw new RuntimeException("Sleep in the retry attempt got interrupted", e); - } - } - retryCount++; - } - String errorMessage = String.format("Exceeded max retry attempts. Failed to execute the Rest API call %s", uri); - log.error(errorMessage); - throw new RuntimeException(errorMessage); - } @VisibleForTesting public void setSleepTimeMultiplier(int multiplier) { diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/CustomRestTemplateConfig.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/CustomRestTemplateConfig.java deleted file mode 100644 index 197ede2ac2..0000000000 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/CustomRestTemplateConfig.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - */ - -package org.opensearch.dataprepper.plugins.source.confluence.rest; - - -import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceSourceConfig; -import org.opensearch.dataprepper.plugins.source.confluence.rest.auth.ConfluenceAuthConfig; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.http.client.ClientHttpRequestInterceptor; -import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; -import org.springframework.web.client.RestTemplate; - -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.OAUTH2; - -@Configuration -public class CustomRestTemplateConfig { - - @Bean - public RestTemplate basicAuthRestTemplate(ConfluenceSourceConfig config, ConfluenceAuthConfig authConfig) { - RestTemplate restTemplate = new RestTemplate(); - restTemplate.setRequestFactory(new HttpComponentsClientHttpRequestFactory()); - ClientHttpRequestInterceptor httpInterceptor; - if (OAUTH2.equals(config.getAuthType())) { - httpInterceptor = new OAuth2RequestInterceptor(authConfig); - } else { - httpInterceptor = new BasicAuthInterceptor(config); - } - restTemplate.getInterceptors().add(httpInterceptor); - return restTemplate; - } - - -} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/OAuth2RequestInterceptor.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/OAuth2RequestInterceptor.java deleted file mode 100644 index 5488fd0850..0000000000 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/OAuth2RequestInterceptor.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - */ - -package org.opensearch.dataprepper.plugins.source.confluence.rest; - -import org.opensearch.dataprepper.plugins.source.confluence.rest.auth.ConfluenceAuthConfig; -import org.opensearch.dataprepper.plugins.source.confluence.rest.auth.ConfluenceOauthConfig; -import org.springframework.http.HttpRequest; -import org.springframework.http.client.ClientHttpRequestExecution; -import org.springframework.http.client.ClientHttpRequestInterceptor; -import org.springframework.http.client.ClientHttpResponse; - -import java.io.IOException; - -public class OAuth2RequestInterceptor implements ClientHttpRequestInterceptor { - - private final ConfluenceAuthConfig config; - - public OAuth2RequestInterceptor(ConfluenceAuthConfig config) { - this.config = config; - } - - @Override - public ClientHttpResponse intercept(HttpRequest request, byte[] body, ClientHttpRequestExecution execution) throws IOException { - request.getHeaders().setBearerAuth(((ConfluenceOauthConfig) config).getAccessToken()); - return execution.execute(request, body); - } - -} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceAuthConfig.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceAuthConfig.java deleted file mode 100644 index 4104601261..0000000000 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceAuthConfig.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - */ - -package org.opensearch.dataprepper.plugins.source.confluence.rest.auth; - -/** - * The interface that defines the behaviour for Jira auth configs. - */ -public interface ConfluenceAuthConfig { - - /** - * Returns the URL for the Jira instance. - * - * @return the URL for the Jira instance. - */ - String getUrl(); - - /** - * Initializes the credentials for the Jira instance. - */ - void initCredentials(); - - /** - * Renews the credentials for the Jira instance. - */ - void renewCredentials(); -} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceAuthFactory.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceAuthFactory.java deleted file mode 100644 index 08b4e929cb..0000000000 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceAuthFactory.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - */ - -package org.opensearch.dataprepper.plugins.source.confluence.rest.auth; - -import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceSourceConfig; -import org.springframework.beans.factory.FactoryBean; -import org.springframework.context.annotation.Configuration; - -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.OAUTH2; - -@Configuration -public class ConfluenceAuthFactory implements FactoryBean { - - private final ConfluenceSourceConfig sourceConfig; - - public ConfluenceAuthFactory(ConfluenceSourceConfig sourceConfig) { - this.sourceConfig = sourceConfig; - } - - @Override - public ConfluenceAuthConfig getObject() { - String authType = sourceConfig.getAuthType(); - if (OAUTH2.equals(authType)) { - return new ConfluenceOauthConfig(sourceConfig); - } - return new ConfluenceBasicAuthConfig(sourceConfig); - } - - @Override - public Class getObjectType() { - return ConfluenceAuthConfig.class; - } -} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceBasicAuthConfig.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceBasicAuthConfig.java deleted file mode 100644 index e400dbf7b0..0000000000 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceBasicAuthConfig.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - */ - -package org.opensearch.dataprepper.plugins.source.confluence.rest.auth; - - -import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceSourceConfig; - -public class ConfluenceBasicAuthConfig implements ConfluenceAuthConfig { - - private String accountUrl; - private final ConfluenceSourceConfig confluenceSourceConfig; - - public ConfluenceBasicAuthConfig(ConfluenceSourceConfig confluenceSourceConfig) { - this.confluenceSourceConfig = confluenceSourceConfig; - accountUrl = confluenceSourceConfig.getAccountUrl(); - if (!accountUrl.endsWith("/")) { - accountUrl += "/"; - } - } - - @Override - public String getUrl() { - return accountUrl; - } - - @Override - public void initCredentials() { - //do nothing for basic authentication - } - - @Override - public void renewCredentials() { - //do nothing for basic authentication - } - - -} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceOauthConfig.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceOauthConfig.java deleted file mode 100644 index 2c1fb08afb..0000000000 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceOauthConfig.java +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - */ - -package org.opensearch.dataprepper.plugins.source.confluence.rest.auth; - -import lombok.Getter; -import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceSourceConfig; -import org.opensearch.dataprepper.plugins.source.confluence.configuration.Oauth2Config; -import org.opensearch.dataprepper.plugins.source.source_crawler.exception.UnAuthorizedException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.http.HttpEntity; -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpMethod; -import org.springframework.http.HttpStatus; -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; -import org.springframework.util.StringUtils; -import org.springframework.web.client.HttpClientErrorException; -import org.springframework.web.client.RestTemplate; - -import java.time.Instant; -import java.util.List; -import java.util.Map; - -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.RETRY_ATTEMPT; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.CqlConstants.SLASH; - -/** - * The type Jira service. - */ - -public class ConfluenceOauthConfig implements ConfluenceAuthConfig { - - public static final String OAuth2_URL = "https://api.atlassian.com/ex/jira/"; - public static final String ACCESSIBLE_RESOURCES = "https://api.atlassian.com/oauth/token/accessible-resources"; - public static final String TOKEN_LOCATION = "https://auth.atlassian.com/oauth/token"; - - public static final String EXPIRES_IN = "expires_in"; - public static final String REFRESH_TOKEN = "refresh_token"; - public static final String ACCESS_TOKEN = "access_token"; - private static final Logger log = LoggerFactory.getLogger(ConfluenceOauthConfig.class); - RestTemplate restTemplate = new RestTemplate(); - private String url; - @Getter - private int expiresInSeconds = 0; - @Getter - private Instant expireTime = Instant.ofEpochMilli(0); - @Getter - private String accessToken; - @Getter - private String refreshToken; - private String cloudId = null; - private final String clientId; - private final String clientSecret; - private final ConfluenceSourceConfig confluenceSourceConfig; - private final Object cloudIdFetchLock = new Object(); - private final Object tokenRenewLock = new Object(); - - public ConfluenceOauthConfig(ConfluenceSourceConfig confluenceSourceConfig) { - this.confluenceSourceConfig = confluenceSourceConfig; - this.accessToken = (String) confluenceSourceConfig.getAuthenticationConfig().getOauth2Config() - .getAccessToken().getValue(); - this.refreshToken = (String) confluenceSourceConfig.getAuthenticationConfig() - .getOauth2Config().getRefreshToken().getValue(); - this.clientId = confluenceSourceConfig.getAuthenticationConfig().getOauth2Config().getClientId(); - this.clientSecret = confluenceSourceConfig.getAuthenticationConfig().getOauth2Config().getClientSecret(); - } - - public String getJiraAccountCloudId() { - log.info("Getting Jira Account Cloud ID"); - synchronized (cloudIdFetchLock) { - if (this.cloudId != null) { - //Someone else must have initialized it - return this.cloudId; - } - - int retryCount = 0; - while (retryCount < RETRY_ATTEMPT) { - retryCount++; - try { - HttpHeaders headers = new HttpHeaders(); - headers.setBearerAuth(accessToken); - headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED); - HttpEntity entity = new HttpEntity<>(headers); - ResponseEntity exchangeResponse = - restTemplate.exchange(ACCESSIBLE_RESOURCES, HttpMethod.GET, entity, Object.class); - List> listResponse = (List>) exchangeResponse.getBody(); - Map response = listResponse.get(0); - this.cloudId = (String) response.get("id"); - return this.cloudId; - } catch (HttpClientErrorException e) { - if (e.getRawStatusCode() == HttpStatus.UNAUTHORIZED.value()) { - renewCredentials(); - } - log.error("Error occurred while accessing resources: ", e); - } - } - throw new UnAuthorizedException(String.format("Access token expired. Unable to renew even after %s attempts", RETRY_ATTEMPT)); - } - } - - public void renewCredentials() { - Instant currentTime = Instant.now(); - if (expireTime.isAfter(currentTime)) { - //There is still time to renew or someone else must have already renewed it - return; - } - - synchronized (tokenRenewLock) { - if (expireTime.isAfter(currentTime)) { - //Someone else must have already renewed it - return; - } - - log.info("Renewing access token and refresh token pair for Jira Connector."); - HttpHeaders headers = new HttpHeaders(); - headers.setContentType(MediaType.APPLICATION_JSON); - String payloadTemplate = "{\"grant_type\": \"%s\", \"client_id\": \"%s\", \"client_secret\": \"%s\", \"refresh_token\": \"%s\"}"; - String payload = String.format(payloadTemplate, "refresh_token", clientId, clientSecret, refreshToken); - HttpEntity entity = new HttpEntity<>(payload, headers); - - Oauth2Config oauth2Config = confluenceSourceConfig.getAuthenticationConfig().getOauth2Config(); - try { - ResponseEntity responseEntity = restTemplate.postForEntity(TOKEN_LOCATION, entity, Map.class); - Map oauthClientResponse = responseEntity.getBody(); - this.accessToken = (String) oauthClientResponse.get(ACCESS_TOKEN); - this.refreshToken = (String) oauthClientResponse.get(REFRESH_TOKEN); - this.expiresInSeconds = (int) oauthClientResponse.get(EXPIRES_IN); - this.expireTime = Instant.now().plusSeconds(expiresInSeconds); - // updating config object's PluginConfigVariable so that it updates the underlying Secret store - oauth2Config.getAccessToken().setValue(this.accessToken); - oauth2Config.getRefreshToken().setValue(this.refreshToken); - log.info("Access Token and Refresh Token pair is now refreshed. Corresponding Secret store key updated."); - } catch (HttpClientErrorException ex) { - this.expireTime = Instant.ofEpochMilli(0); - this.expiresInSeconds = 0; - HttpStatus statusCode = ex.getStatusCode(); - log.error("Failed to renew access token. Status code: {}, Error Message: {}", - statusCode, ex.getMessage()); - if (statusCode == HttpStatus.FORBIDDEN || statusCode == HttpStatus.UNAUTHORIZED) { - log.info("Trying to refresh the secrets"); - // Refreshing the secrets. It should help if someone already renewed the tokens. - // Refreshing one of the secret refreshes the entire store so triggering refresh on just one - oauth2Config.getAccessToken().refresh(); - this.accessToken = (String) oauth2Config.getAccessToken().getValue(); - this.refreshToken = (String) oauth2Config.getRefreshToken().getValue(); - this.expireTime = Instant.now().plusSeconds(10); - } - throw new RuntimeException("Failed to renew access token message:" + ex.getMessage(), ex); - } - } - } - - @Override - public String getUrl() { - if (!StringUtils.hasLength(url)) { - synchronized (cloudIdFetchLock) { - if (!StringUtils.hasLength(url)) { - initCredentials(); - } - } - } - return url; - } - - /** - * Method for getting Jira url based on auth type. - */ - @Override - public void initCredentials() { - //For OAuth based flow, we use a different Jira url - this.cloudId = getJiraAccountCloudId(); - this.url = OAuth2_URL + this.cloudId + SLASH; - } -} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/Constants.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/Constants.java index 114c33cebd..e63d001a87 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/Constants.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/Constants.java @@ -27,6 +27,5 @@ public class Constants { public static final String CONTENT_ID = "id"; public static final String CREATED = "created"; public static final String BASIC = "Basic"; - public static final String LIVE = "live"; public static final String PLUGIN_NAME = "confluence"; } \ No newline at end of file diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClientTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClientTest.java index 27a0885208..ea7f35350e 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClientTest.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClientTest.java @@ -20,13 +20,13 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.plugins.source.atlassian.rest.auth.AtlassianAuthConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.rest.auth.AtlassianAuthFactory; import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceServiceTest; import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceSourceConfig; -import org.opensearch.dataprepper.plugins.source.confluence.exception.BadRequestException; -import org.opensearch.dataprepper.plugins.source.confluence.exception.UnAuthorizedException; import org.opensearch.dataprepper.plugins.source.confluence.models.ConfluenceSearchResults; -import org.opensearch.dataprepper.plugins.source.confluence.rest.auth.ConfluenceAuthConfig; -import org.opensearch.dataprepper.plugins.source.confluence.rest.auth.ConfluenceAuthFactory; +import org.opensearch.dataprepper.plugins.source.source_crawler.exception.BadRequestException; +import org.opensearch.dataprepper.plugins.source.source_crawler.exception.UnAuthorizedException; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.client.HttpClientErrorException; @@ -57,7 +57,7 @@ public class ConfluenceRestClientTest { private RestTemplate restTemplate; @Mock - private ConfluenceAuthConfig authConfig; + private AtlassianAuthConfig authConfig; private final PluginMetrics pluginMetrics = PluginMetrics.fromNames("jiraRestClient", "aws"); @@ -76,7 +76,7 @@ public void testFetchingJiraIssue(String configFileName) { String exampleTicketResponse = "{\"id\":\"123\",\"key\":\"key\",\"self\":\"https://example.com/rest/api/2/issue/123\"}"; doReturn(new ResponseEntity<>(exampleTicketResponse, HttpStatus.OK)).when(restTemplate).getForEntity(any(URI.class), any(Class.class)); ConfluenceSourceConfig confluenceSourceConfig = ConfluenceServiceTest.createJiraConfigurationFromYaml(configFileName); - ConfluenceAuthConfig authConfig = new ConfluenceAuthFactory(confluenceSourceConfig).getObject(); + AtlassianAuthConfig authConfig = new AtlassianAuthFactory(confluenceSourceConfig).getObject(); ConfluenceRestClient confluenceRestClient = new ConfluenceRestClient(restTemplate, authConfig, pluginMetrics); String ticketDetails = confluenceRestClient.getContent("key"); assertEquals(exampleTicketResponse, ticketDetails); From 66befd51a88f06826ebf5f533a99dc0b573cc9c0 Mon Sep 17 00:00:00 2001 From: Santhosh Gandhe <1909520+san81@users.noreply.github.com> Date: Mon, 3 Feb 2025 16:50:35 -0800 Subject: [PATCH 08/14] test cases in a running state Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com> --- .../rest/BasicAuthInterceptorTest.java | 11 +- .../rest/OAuth2RequestInterceptorTest.java | 6 +- .../rest/auth/AtlassianAuthFactoryTest.java} | 24 +-- .../auth/AtlassianBasicAuthConfigTest.java} | 12 +- .../rest/auth/AtlassianOauthConfigTest.java} | 30 ++-- .../atlassian/utils/ConfigUtilForTests.java | 45 ++++++ .../utils/MockPluginConfigVariableImpl.java | 44 ++++++ .../resources/basic-auth-jira-pipeline.yaml | 0 .../resources/oauth2-auth-jira-pipeline.yaml | 0 .../confluence-source/build.gradle | 1 + .../confluence/rest/ConfluenceRestClient.java | 7 - .../ConfluenceConfigHelperTest.java | 6 +- .../confluence/ConfluenceServiceTest.java | 141 +++++++----------- .../ConfluenceSourceConfigTest.java | 40 +++-- .../confluence/ConfluenceSourceTest.java | 10 +- .../confluence/models/ConfluenceItemTest.java | 27 ++-- .../rest/ConfluenceRestClientTest.java | 27 ++-- .../rest/CustomRestTemplateConfigTest.java | 13 +- .../basic-auth-confluence-pipeline.yaml | 6 + .../oauth2-auth-confluence-pipeline.yaml | 6 + 20 files changed, 244 insertions(+), 212 deletions(-) rename data-prepper-plugins/saas-source-plugins/{confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence => atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian}/rest/BasicAuthInterceptorTest.java (84%) rename data-prepper-plugins/saas-source-plugins/{confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence => atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian}/rest/OAuth2RequestInterceptorTest.java (90%) rename data-prepper-plugins/saas-source-plugins/{confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceAuthFactoryTest.java => atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianAuthFactoryTest.java} (65%) rename data-prepper-plugins/saas-source-plugins/{confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/JiraBasicAuthConfigTest.java => atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianBasicAuthConfigTest.java} (71%) rename data-prepper-plugins/saas-source-plugins/{confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/JiraOauthConfigTest.java => atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianOauthConfigTest.java} (86%) create mode 100644 data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/utils/ConfigUtilForTests.java create mode 100644 data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/utils/MockPluginConfigVariableImpl.java rename data-prepper-plugins/saas-source-plugins/{confluence-source => atlassian-commons}/src/test/resources/basic-auth-jira-pipeline.yaml (100%) rename data-prepper-plugins/saas-source-plugins/{confluence-source => atlassian-commons}/src/test/resources/oauth2-auth-jira-pipeline.yaml (100%) create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/test/resources/basic-auth-confluence-pipeline.yaml create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/test/resources/oauth2-auth-confluence-pipeline.yaml diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/BasicAuthInterceptorTest.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/BasicAuthInterceptorTest.java similarity index 84% rename from data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/BasicAuthInterceptorTest.java rename to data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/BasicAuthInterceptorTest.java index 68d05b47aa..b88d887e42 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/BasicAuthInterceptorTest.java +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/BasicAuthInterceptorTest.java @@ -8,17 +8,16 @@ * */ -package org.opensearch.dataprepper.plugins.source.confluence.rest; +package org.opensearch.dataprepper.plugins.source.atlassian.rest; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceSourceConfig; -import org.opensearch.dataprepper.plugins.source.confluence.configuration.AuthenticationConfig; -import org.opensearch.dataprepper.plugins.source.confluence.configuration.BasicConfig; -import org.opensearch.dataprepper.plugins.source.confluence.rest.BasicAuthInterceptor; +import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.configuration.AuthenticationConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.configuration.BasicConfig; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpRequest; import org.springframework.http.client.ClientHttpRequestExecution; @@ -48,7 +47,7 @@ public class BasicAuthInterceptorTest { @Mock private ClientHttpResponse mockResponse; @Mock - private ConfluenceSourceConfig mockConfig; + private AtlassianSourceConfig mockConfig; @Mock private HttpHeaders mockHeaders; diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/OAuth2RequestInterceptorTest.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/OAuth2RequestInterceptorTest.java similarity index 90% rename from data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/OAuth2RequestInterceptorTest.java rename to data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/OAuth2RequestInterceptorTest.java index 554b97e674..b1b6f51b3f 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/OAuth2RequestInterceptorTest.java +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/OAuth2RequestInterceptorTest.java @@ -8,14 +8,14 @@ * */ -package org.opensearch.dataprepper.plugins.source.confluence.rest; +package org.opensearch.dataprepper.plugins.source.atlassian.rest; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.dataprepper.plugins.source.confluence.rest.auth.ConfluenceOauthConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.rest.auth.AtlassianOauthConfig; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpRequest; import org.springframework.http.client.ClientHttpRequestExecution; @@ -41,7 +41,7 @@ public class OAuth2RequestInterceptorTest { private ClientHttpResponse mockResponse; @Mock - private ConfluenceOauthConfig mockConfig; + private AtlassianOauthConfig mockConfig; @Mock private HttpHeaders mockHeaders; diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceAuthFactoryTest.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianAuthFactoryTest.java similarity index 65% rename from data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceAuthFactoryTest.java rename to data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianAuthFactoryTest.java index 60fdbdcfd1..7dc618f744 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceAuthFactoryTest.java +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianAuthFactoryTest.java @@ -8,7 +8,7 @@ * */ -package org.opensearch.dataprepper.plugins.source.confluence.rest.auth; +package org.opensearch.dataprepper.plugins.source.atlassian.rest.auth; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -16,20 +16,20 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.dataprepper.model.plugin.PluginConfigVariable; -import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceSourceConfig; -import org.opensearch.dataprepper.plugins.source.confluence.configuration.AuthenticationConfig; -import org.opensearch.dataprepper.plugins.source.confluence.configuration.Oauth2Config; +import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.configuration.AuthenticationConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.configuration.Oauth2Config; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.mockito.Mockito.when; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.OAUTH2; +import static org.opensearch.dataprepper.plugins.source.atlassian.utils.Constants.OAUTH2; @ExtendWith(MockitoExtension.class) -public class ConfluenceAuthFactoryTest { +public class AtlassianAuthFactoryTest { @Mock - private ConfluenceSourceConfig sourceConfig; + private AtlassianSourceConfig sourceConfig; @Mock private AuthenticationConfig authenticationConfig; @@ -43,11 +43,11 @@ public class ConfluenceAuthFactoryTest { @Mock private PluginConfigVariable refreshTokenPluginConfigVariable; - private ConfluenceAuthFactory confluenceAuthFactory; + private AtlassianAuthFactory confluenceAuthFactory; @BeforeEach void setUp() { - confluenceAuthFactory = new ConfluenceAuthFactory(sourceConfig); + confluenceAuthFactory = new AtlassianAuthFactory(sourceConfig); } @Test @@ -58,17 +58,17 @@ void testGetObjectOauth2() { when(oauth2Config.getRefreshToken()).thenReturn(refreshTokenPluginConfigVariable); when(oauth2Config.getAccessToken()).thenReturn(accessTokenPluginConfigVariable); when(accessTokenPluginConfigVariable.getValue()).thenReturn("mockRefreshToken"); - assertInstanceOf(ConfluenceOauthConfig.class, confluenceAuthFactory.getObject()); + assertInstanceOf(AtlassianOauthConfig.class, confluenceAuthFactory.getObject()); } @Test void testGetObjectBasicAuth() { when(sourceConfig.getAccountUrl()).thenReturn("https://example.com"); - assertInstanceOf(ConfluenceBasicAuthConfig.class, confluenceAuthFactory.getObject()); + assertInstanceOf(AtlassianBasicAuthConfig.class, confluenceAuthFactory.getObject()); } @Test void testGetObjectType() { - assertEquals(ConfluenceAuthConfig.class, confluenceAuthFactory.getObjectType()); + assertEquals(AtlassianAuthConfig.class, confluenceAuthFactory.getObjectType()); } } diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/JiraBasicAuthConfigTest.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianBasicAuthConfigTest.java similarity index 71% rename from data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/JiraBasicAuthConfigTest.java rename to data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianBasicAuthConfigTest.java index ae6f8f3325..7c0a8196af 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/JiraBasicAuthConfigTest.java +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianBasicAuthConfigTest.java @@ -8,30 +8,30 @@ * */ -package org.opensearch.dataprepper.plugins.source.confluence.rest.auth; +package org.opensearch.dataprepper.plugins.source.atlassian.rest.auth; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceSourceConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) -public class JiraBasicAuthConfigTest { +public class AtlassianBasicAuthConfigTest { String url = "https://example.com"; @Mock - private ConfluenceSourceConfig confluenceSourceConfig; - private ConfluenceBasicAuthConfig jiraBasicAuthConfig; + private AtlassianSourceConfig confluenceSourceConfig; + private AtlassianBasicAuthConfig jiraBasicAuthConfig; @BeforeEach void setUp() { when(confluenceSourceConfig.getAccountUrl()).thenReturn(url); - jiraBasicAuthConfig = new ConfluenceBasicAuthConfig(confluenceSourceConfig); + jiraBasicAuthConfig = new AtlassianBasicAuthConfig(confluenceSourceConfig); } @Test diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/JiraOauthConfigTest.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianOauthConfigTest.java similarity index 86% rename from data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/JiraOauthConfigTest.java rename to data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianOauthConfigTest.java index 7d7ef8aae4..3131e5e502 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/JiraOauthConfigTest.java +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianOauthConfigTest.java @@ -8,7 +8,7 @@ * */ -package org.opensearch.dataprepper.plugins.source.confluence.rest.auth; +package org.opensearch.dataprepper.plugins.source.atlassian.rest.auth; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -16,10 +16,10 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.dataprepper.model.plugin.PluginConfigVariable; -import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceServiceTest; -import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceSourceConfig; -import org.opensearch.dataprepper.plugins.source.confluence.configuration.Oauth2Config; -import org.opensearch.dataprepper.plugins.source.confluence.exception.UnAuthorizedException; +import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.configuration.Oauth2Config; +import org.opensearch.dataprepper.plugins.source.atlassian.utils.ConfigUtilForTests; +import org.opensearch.dataprepper.plugins.source.source_crawler.exception.UnAuthorizedException; import org.opensearch.dataprepper.test.helper.ReflectivelySetField; import org.springframework.http.HttpEntity; import org.springframework.http.HttpMethod; @@ -44,23 +44,23 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.RETRY_ATTEMPT; +import static org.opensearch.dataprepper.plugins.source.atlassian.utils.Constants.RETRY_ATTEMPT; @ExtendWith(MockitoExtension.class) -public class JiraOauthConfigTest { +public class AtlassianOauthConfigTest { @Mock RestTemplate restTemplateMock; - ConfluenceSourceConfig confluenceSourceConfig; + AtlassianSourceConfig confluenceSourceConfig; @Mock PluginConfigVariable accessTokenVariable; @BeforeEach void setUp() { - confluenceSourceConfig = ConfluenceServiceTest.createJiraConfigurationFromYaml("oauth2-auth-jira-pipeline.yaml"); + confluenceSourceConfig = ConfigUtilForTests.createJiraConfigurationFromYaml("oauth2-auth-jira-pipeline.yaml"); } @Test @@ -69,7 +69,7 @@ void testRenewToken() throws InterruptedException { Map firstMockResponseMap = Map.of("access_token", "first_mock_access_token", "refresh_token", "first_mock_refresh_token", "expires_in", 3600); - ConfluenceOauthConfig jiraOauthConfig = new ConfluenceOauthConfig(confluenceSourceConfig); + AtlassianOauthConfig jiraOauthConfig = new AtlassianOauthConfig(confluenceSourceConfig); when(restTemplateMock.postForEntity(any(String.class), any(HttpEntity.class), any(Class.class))) .thenReturn(new ResponseEntity<>(firstMockResponseMap, HttpStatus.OK)); jiraOauthConfig.restTemplate = restTemplateMock; @@ -96,7 +96,7 @@ void testRenewToken() throws InterruptedException { @Test void testFailedToRenewAccessToken() throws NoSuchFieldException, IllegalAccessException { - ConfluenceOauthConfig jiraOauthConfig = new ConfluenceOauthConfig(confluenceSourceConfig); + AtlassianOauthConfig jiraOauthConfig = new AtlassianOauthConfig(confluenceSourceConfig); Oauth2Config oauth2Config = confluenceSourceConfig.getAuthenticationConfig().getOauth2Config(); ReflectivelySetField.setField(Oauth2Config.class, oauth2Config, "accessToken", accessTokenVariable); when(restTemplateMock.postForEntity(any(String.class), any(HttpEntity.class), any(Class.class))) @@ -110,7 +110,7 @@ void testFailedToRenewAccessToken() throws NoSuchFieldException, IllegalAccessEx @Test void testFailedToRenewAccessToken_with_unauthorized_and_trigger_secrets_refresh() throws NoSuchFieldException, IllegalAccessException { - ConfluenceOauthConfig jiraOauthConfig = new ConfluenceOauthConfig(confluenceSourceConfig); + AtlassianOauthConfig jiraOauthConfig = new AtlassianOauthConfig(confluenceSourceConfig); Oauth2Config oauth2Config = confluenceSourceConfig.getAuthenticationConfig().getOauth2Config(); ReflectivelySetField.setField(Oauth2Config.class, oauth2Config, "accessToken", accessTokenVariable); HttpClientErrorException unAuthorizedException = new HttpClientErrorException(HttpStatus.UNAUTHORIZED); @@ -129,7 +129,7 @@ void testGetJiraAccountCloudId() throws InterruptedException { mockGetCallResponse.put("id", "test_cloud_id"); when(restTemplateMock.exchange(any(String.class), any(HttpMethod.class), any(HttpEntity.class), any(Class.class))) .thenReturn(new ResponseEntity<>(List.of(mockGetCallResponse), HttpStatus.OK)); - ConfluenceOauthConfig jiraOauthConfig = new ConfluenceOauthConfig(confluenceSourceConfig); + AtlassianOauthConfig jiraOauthConfig = new AtlassianOauthConfig(confluenceSourceConfig); jiraOauthConfig.restTemplate = restTemplateMock; ExecutorService executor = Executors.newFixedThreadPool(2); @@ -159,7 +159,7 @@ void testGetJiraAccountCloudIdUnauthorizedCase() { "expires_in", 3600); when(restTemplateMock.postForEntity(any(String.class), any(HttpEntity.class), any(Class.class))) .thenReturn(new ResponseEntity<>(mockRenewTokenResponse, HttpStatus.OK)); - ConfluenceOauthConfig jiraOauthConfig = new ConfluenceOauthConfig(confluenceSourceConfig); + AtlassianOauthConfig jiraOauthConfig = new AtlassianOauthConfig(confluenceSourceConfig); jiraOauthConfig.restTemplate = restTemplateMock; @@ -176,7 +176,7 @@ void testFailedToGetCloudId() { when(restTemplateMock.exchange(any(String.class), any(HttpMethod.class), any(HttpEntity.class), any(Class.class))) .thenThrow(new HttpClientErrorException(HttpStatus.UNAUTHORIZED)) .thenThrow(HttpClientErrorException.class); - ConfluenceOauthConfig jiraOauthConfig = new ConfluenceOauthConfig(confluenceSourceConfig); + AtlassianOauthConfig jiraOauthConfig = new AtlassianOauthConfig(confluenceSourceConfig); jiraOauthConfig.restTemplate = restTemplateMock; assertThrows(RuntimeException.class, jiraOauthConfig::getUrl); for (int i = 0; i <= RETRY_ATTEMPT; i++) { diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/utils/ConfigUtilForTests.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/utils/ConfigUtilForTests.java new file mode 100644 index 0000000000..61d1889935 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/utils/ConfigUtilForTests.java @@ -0,0 +1,45 @@ +package org.opensearch.dataprepper.plugins.source.atlassian.utils; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.configuration.Oauth2Config; +import org.opensearch.dataprepper.test.helper.ReflectivelySetField; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.InputStream; + +public class ConfigUtilForTests { + + public static final Logger log = LoggerFactory.getLogger(ConfigUtilForTests.class); + + private static InputStream getResourceAsStream(String resourceName) { + InputStream inputStream = Thread.currentThread().getContextClassLoader().getResourceAsStream(resourceName); + if (inputStream == null) { + inputStream = ConfigUtilForTests.class.getResourceAsStream("/" + resourceName); + } + return inputStream; + } + + public static AtlassianSourceConfig createJiraConfigurationFromYaml(String fileName) { + ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory()); + try (InputStream inputStream = getResourceAsStream(fileName)) { + AtlassianSourceConfig confluenceSourceConfig = objectMapper.readValue(inputStream, AtlassianSourceConfig.class); + Oauth2Config oauth2Config = confluenceSourceConfig.getAuthenticationConfig().getOauth2Config(); + if (oauth2Config != null) { + ReflectivelySetField.setField(Oauth2Config.class, oauth2Config, "accessToken", + new MockPluginConfigVariableImpl("mockAccessToken")); + ReflectivelySetField.setField(Oauth2Config.class, oauth2Config, "refreshToken", + new MockPluginConfigVariableImpl("mockRefreshToken")); + } + return confluenceSourceConfig; + } catch (IOException ex) { + log.error("Failed to parse pipeline Yaml", ex); + } catch (Exception e) { + throw new RuntimeException(e); + } + return null; + } +} diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/utils/MockPluginConfigVariableImpl.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/utils/MockPluginConfigVariableImpl.java new file mode 100644 index 0000000000..4c5f4e8e93 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/utils/MockPluginConfigVariableImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.atlassian.utils; + +import org.opensearch.dataprepper.model.plugin.PluginConfigVariable; + +/** + * Mock implementation of PluginConfigVariable interface used only for Unit Testing. + */ +public class MockPluginConfigVariableImpl implements PluginConfigVariable { + + private Object defaultValue; + + public MockPluginConfigVariableImpl(Object defaultValue) { + this.defaultValue = defaultValue; + } + + @Override + public Object getValue() { + return null; + } + + @Override + public void setValue(Object someValue) { + this.defaultValue = someValue; + } + + @Override + public void refresh() { + } + + @Override + public boolean isUpdatable() { + return true; + } +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/resources/basic-auth-jira-pipeline.yaml b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/resources/basic-auth-jira-pipeline.yaml similarity index 100% rename from data-prepper-plugins/saas-source-plugins/confluence-source/src/test/resources/basic-auth-jira-pipeline.yaml rename to data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/resources/basic-auth-jira-pipeline.yaml diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/resources/oauth2-auth-jira-pipeline.yaml b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/resources/oauth2-auth-jira-pipeline.yaml similarity index 100% rename from data-prepper-plugins/saas-source-plugins/confluence-source/src/test/resources/oauth2-auth-jira-pipeline.yaml rename to data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/resources/oauth2-auth-jira-pipeline.yaml diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/build.gradle b/data-prepper-plugins/saas-source-plugins/confluence-source/build.gradle index 07308a50f7..cd1cee26e8 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/build.gradle +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/build.gradle @@ -23,6 +23,7 @@ dependencies { annotationProcessor 'org.projectlombok:lombok:1.18.30' testImplementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.4' + testImplementation project(path: ':data-prepper-plugins:saas-source-plugins:atlassian-commons') testImplementation project(path: ':data-prepper-test-common') implementation(libs.spring.context) { diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClient.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClient.java index bcda06d076..d3dcdad2d8 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClient.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClient.java @@ -10,7 +10,6 @@ package org.opensearch.dataprepper.plugins.source.confluence.rest; -import com.google.common.annotations.VisibleForTesting; import io.micrometer.core.annotation.Timed; import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.Timer; @@ -44,7 +43,6 @@ public class ConfluenceRestClient extends AtlassianRestClient { private static final String SEARCH_CALL_LATENCY_TIMER = "searchCallLatency"; private static final String SPACES_FETCH_LATENCY_TIMER = "spacesFetchLatency"; private static final String PAGES_REQUESTED = "pagesRequested"; - private int sleepTimeMultiplier = 1000; private final RestTemplate restTemplate; private final AtlassianAuthConfig authConfig; private final Timer contentFetchLatencyTimer; @@ -99,9 +97,4 @@ public String getContent(String contentId) { return invokeRestApi(uri, String.class).getBody(); } - - @VisibleForTesting - public void setSleepTimeMultiplier(int multiplier) { - sleepTimeMultiplier = multiplier; - } } diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceConfigHelperTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceConfigHelperTest.java index 57ea307b5c..80528646e4 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceConfigHelperTest.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceConfigHelperTest.java @@ -15,11 +15,11 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.dataprepper.model.plugin.PluginConfigVariable; -import org.opensearch.dataprepper.plugins.source.confluence.configuration.AuthenticationConfig; -import org.opensearch.dataprepper.plugins.source.confluence.configuration.BasicConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.configuration.AuthenticationConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.configuration.BasicConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.configuration.Oauth2Config; import org.opensearch.dataprepper.plugins.source.confluence.configuration.FilterConfig; import org.opensearch.dataprepper.plugins.source.confluence.configuration.NameConfig; -import org.opensearch.dataprepper.plugins.source.confluence.configuration.Oauth2Config; import org.opensearch.dataprepper.plugins.source.confluence.configuration.PageTypeConfig; import org.opensearch.dataprepper.plugins.source.confluence.configuration.SpaceConfig; import org.opensearch.dataprepper.plugins.source.confluence.utils.ConfluenceConfigHelper; diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceServiceTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceServiceTest.java index d966b2e3ad..c80f5cc78f 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceServiceTest.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceServiceTest.java @@ -20,13 +20,14 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.plugin.PluginConfigVariable; -import org.opensearch.dataprepper.plugins.source.confluence.configuration.Oauth2Config; -import org.opensearch.dataprepper.plugins.source.confluence.exception.BadRequestException; +import org.opensearch.dataprepper.plugins.source.atlassian.configuration.Oauth2Config; import org.opensearch.dataprepper.plugins.source.confluence.models.ConfluenceItem; import org.opensearch.dataprepper.plugins.source.confluence.models.ConfluenceSearchResults; +import org.opensearch.dataprepper.plugins.source.confluence.models.SpaceItem; import org.opensearch.dataprepper.plugins.source.confluence.rest.ConfluenceRestClient; import org.opensearch.dataprepper.plugins.source.confluence.utils.MockPluginConfigVariableImpl; import org.opensearch.dataprepper.plugins.source.source_crawler.base.PluginExecutorServiceProvider; +import org.opensearch.dataprepper.plugins.source.source_crawler.exception.BadRequestException; import org.opensearch.dataprepper.plugins.source.source_crawler.model.ItemInfo; import org.opensearch.dataprepper.test.helper.ReflectivelySetField; import org.slf4j.Logger; @@ -40,13 +41,13 @@ import java.util.List; import java.util.Map; import java.util.Queue; +import java.util.Random; import java.util.UUID; import java.util.concurrent.ConcurrentLinkedQueue; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyString; @@ -55,14 +56,9 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; -import static org.opensearch.dataprepper.plugins.source.confluence.rest.auth.ConfluenceOauthConfig.ACCESSIBLE_RESOURCES; +import static org.opensearch.dataprepper.plugins.source.atlassian.rest.auth.AtlassianOauthConfig.ACCESSIBLE_RESOURCES; import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.BASIC; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.CREATED; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.KEY; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.LAST_MODIFIED; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.NAME; import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.OAUTH2; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.SPACE; /** @@ -85,7 +81,7 @@ private static InputStream getResourceAsStream(String resourceName) { return inputStream; } - public static ConfluenceSourceConfig createJiraConfigurationFromYaml(String fileName) { + public static ConfluenceSourceConfig createConfluenceConfigurationFromYaml(String fileName) { ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory()); try (InputStream inputStream = getResourceAsStream(fileName)) { ConfluenceSourceConfig confluenceSourceConfig = objectMapper.readValue(inputStream, ConfluenceSourceConfig.class); @@ -105,10 +101,9 @@ public static ConfluenceSourceConfig createJiraConfigurationFromYaml(String file return null; } - public static ConfluenceSourceConfig createJiraConfiguration(String auth_type, - List issueType, - List issueStatus, - List projectKey) throws JsonProcessingException { + public static ConfluenceSourceConfig createConfluenceConfiguration(String auth_type, + List issueType, + List projectKey) throws JsonProcessingException { PluginConfigVariable pcvAccessToken = null; PluginConfigVariable pcvRefreshToken = null; ObjectMapper objectMapper = new ObjectMapper(); @@ -132,20 +127,16 @@ public static ConfluenceSourceConfig createJiraConfiguration(String auth_type, hosts.add(ACCESSIBLE_RESOURCES); Map filterMap = new HashMap<>(); - Map projectMap = new HashMap<>(); - Map issueTypeMap = new HashMap<>(); - Map statusMap = new HashMap<>(); + Map spacesMap = new HashMap<>(); + Map contentTypeMap = new HashMap<>(); - issueTypeMap.put("include", issueType); - filterMap.put("issue_type", issueTypeMap); - - statusMap.put("include", issueStatus); - filterMap.put("status", statusMap); + contentTypeMap.put("include", issueType); + filterMap.put("page_type", contentTypeMap); Map nameMap = new HashMap<>(); nameMap.put("include", projectKey); - projectMap.put("key", nameMap); - filterMap.put("project", projectMap); + spacesMap.put("key", nameMap); + filterMap.put("space", spacesMap); jiraSourceConfigMap.put("hosts", hosts); @@ -174,10 +165,9 @@ void tearDown() { @Test void testJiraServiceInitialization() throws JsonProcessingException { - List issueType = new ArrayList<>(); - List issueStatus = new ArrayList<>(); - List projectKey = new ArrayList<>(); - ConfluenceSourceConfig confluenceSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey); + List contentType = new ArrayList<>(); + List spacesKey = new ArrayList<>(); + ConfluenceSourceConfig confluenceSourceConfig = createConfluenceConfiguration(BASIC, contentType, spacesKey); ConfluenceService confluenceService = new ConfluenceService(confluenceSourceConfig, confluenceRestClient, pluginMetrics); assertNotNull(confluenceService); when(confluenceRestClient.getContent(anyString())).thenReturn("test String"); @@ -186,55 +176,54 @@ void testJiraServiceInitialization() throws JsonProcessingException { @Test public void testGetPages() throws JsonProcessingException { - List issueType = new ArrayList<>(); - List issueStatus = new ArrayList<>(); - List projectKey = new ArrayList<>(); - issueType.add("Task"); - issueStatus.add("Done"); - projectKey.add("KAN"); - ConfluenceSourceConfig confluenceSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey); + List contentType = new ArrayList<>(); + List spaceKey = new ArrayList<>(); + contentType.add("page"); + spaceKey.add("KAN"); + ConfluenceSourceConfig confluenceSourceConfig = createConfluenceConfiguration(BASIC, contentType, spaceKey); ConfluenceService confluenceService = spy(new ConfluenceService(confluenceSourceConfig, confluenceRestClient, pluginMetrics)); - List mockIssues = new ArrayList<>(); - ConfluenceItem issue1 = createConfluenceItemBean(false, false); - mockIssues.add(issue1); - ConfluenceItem issue2 = createConfluenceItemBean(true, false); - mockIssues.add(issue2); - ConfluenceItem issue3 = createConfluenceItemBean(false, true); - mockIssues.add(issue3); + List mockPages = new ArrayList<>(); + ConfluenceItem issue1 = createConfluenceItemBean(); + mockPages.add(issue1); + ConfluenceItem issue2 = createConfluenceItemBean(); + mockPages.add(issue2); + ConfluenceItem issue3 = createConfluenceItemBean(); + mockPages.add(issue3); ConfluenceSearchResults mockConfluenceSearchResults = mock(ConfluenceSearchResults.class); + when(mockConfluenceSearchResults.getResults()).thenReturn(mockPages); doReturn(mockConfluenceSearchResults).when(confluenceRestClient).getAllContent(any(StringBuilder.class), anyInt()); Instant timestamp = Instant.ofEpochSecond(0); Queue itemInfoQueue = new ConcurrentLinkedQueue<>(); confluenceService.getPages(confluenceSourceConfig, timestamp, itemInfoQueue); - assertEquals(mockIssues.size(), itemInfoQueue.size()); + assertEquals(mockPages.size(), itemInfoQueue.size()); } @Test public void buildIssueItemInfoMultipleFutureThreads() throws JsonProcessingException { List issueType = new ArrayList<>(); - List issueStatus = new ArrayList<>(); List projectKey = new ArrayList<>(); issueType.add("Task"); - ConfluenceSourceConfig confluenceSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey); + ConfluenceSourceConfig confluenceSourceConfig = createConfluenceConfiguration(BASIC, issueType, projectKey); ConfluenceService confluenceService = spy(new ConfluenceService(confluenceSourceConfig, confluenceRestClient, pluginMetrics)); List mockIssues = new ArrayList<>(); - for (int i = 0; i < 50; i++) { - ConfluenceItem issue1 = createConfluenceItemBean(false, false); - mockIssues.add(issue1); + Random random = new Random(); + int numberOfIssues = random.nextInt(100); + for (int i = 0; i < numberOfIssues; i++) { + mockIssues.add(createConfluenceItemBean()); } ConfluenceSearchResults mockConfluenceSearchResults = mock(ConfluenceSearchResults.class); - + when(mockConfluenceSearchResults.getResults()).thenReturn(mockIssues); doReturn(mockConfluenceSearchResults).when(confluenceRestClient).getAllContent(any(StringBuilder.class), anyInt()); Instant timestamp = Instant.ofEpochSecond(0); Queue itemInfoQueue = new ConcurrentLinkedQueue<>(); confluenceService.getPages(confluenceSourceConfig, timestamp, itemInfoQueue); - assertTrue(itemInfoQueue.size() >= 100); + assertEquals(numberOfIssues, itemInfoQueue.size()); } @Test @@ -249,7 +238,7 @@ public void testBadProjectKeys() throws JsonProcessingException { projectKey.add("!@#$"); projectKey.add("AAAAAAAAAAAAAA"); - ConfluenceSourceConfig confluenceSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey); + ConfluenceSourceConfig confluenceSourceConfig = createConfluenceConfiguration(BASIC, issueType, projectKey); ConfluenceService confluenceService = new ConfluenceService(confluenceSourceConfig, confluenceRestClient, pluginMetrics); Instant timestamp = Instant.ofEpochSecond(0); @@ -264,7 +253,7 @@ public void testGetPagesException() throws JsonProcessingException { List issueStatus = new ArrayList<>(); List projectKey = new ArrayList<>(); issueType.add("Task"); - ConfluenceSourceConfig confluenceSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey); + ConfluenceSourceConfig confluenceSourceConfig = createConfluenceConfiguration(BASIC, issueType, projectKey); ConfluenceService confluenceService = spy(new ConfluenceService(confluenceSourceConfig, confluenceRestClient, pluginMetrics)); doThrow(RuntimeException.class).when(confluenceRestClient).getAllContent(any(StringBuilder.class), anyInt()); @@ -276,45 +265,15 @@ public void testGetPagesException() throws JsonProcessingException { } - private ConfluenceItem createConfluenceItemBean(boolean nullFields, boolean createdNull) { - ConfluenceItem issue1 = new ConfluenceItem(); - issue1.setId(UUID.randomUUID().toString()); - issue1.setTitle("issue_1_key"); - - Map fieldMap = new HashMap<>(); - if (!nullFields) { - fieldMap.put(CREATED, "2024-07-06T21:12:23.437-0700"); - fieldMap.put(LAST_MODIFIED, "2024-07-06T21:12:23.106-0700"); - } else { - fieldMap.put(CREATED, 0); - fieldMap.put(LAST_MODIFIED, 0); - } - if (createdNull) { - fieldMap.put(CREATED, null); - } - - Map issueTypeMap = new HashMap<>(); - issueTypeMap.put("name", "Task"); - issueTypeMap.put("self", "https://example.com/rest/api/2/issuetype/1"); - issueTypeMap.put("id", "1"); - fieldMap.put("issuetype", issueTypeMap); - - Map projectMap = new HashMap<>(); - if (!nullFields) { - projectMap.put(NAME, "project name test"); - projectMap.put(KEY, "TEST"); - } - fieldMap.put(SPACE, projectMap); - - Map priorityMap = new HashMap<>(); - priorityMap.put(NAME, "Medium"); - fieldMap.put("priority", priorityMap); - - Map statusMap = new HashMap<>(); - statusMap.put(NAME, "In Progress"); - fieldMap.put("statuses", statusMap); - - return issue1; + private ConfluenceItem createConfluenceItemBean() { + ConfluenceItem confluenceItem = new ConfluenceItem(); + confluenceItem.setId(UUID.randomUUID().toString()); + confluenceItem.setTitle("issue_1_key"); + SpaceItem spaceItem = new SpaceItem(); + spaceItem.setId(new Random().nextInt()); + spaceItem.setKey(UUID.randomUUID().toString()); + confluenceItem.setSpaceItem(spaceItem); + return confluenceItem; } } \ No newline at end of file diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceConfigTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceConfigTest.java index 73fdbe9dbe..69535ae697 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceConfigTest.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceConfigTest.java @@ -13,7 +13,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.jupiter.api.Test; import org.opensearch.dataprepper.model.plugin.PluginConfigVariable; -import org.opensearch.dataprepper.plugins.source.confluence.configuration.Oauth2Config; +import org.opensearch.dataprepper.plugins.source.atlassian.configuration.Oauth2Config; import org.opensearch.dataprepper.plugins.source.confluence.utils.MockPluginConfigVariableImpl; import org.opensearch.dataprepper.test.helper.ReflectivelySetField; @@ -36,11 +36,10 @@ public class ConfluenceSourceConfigTest { private final String password = "test Jira Credential"; private final String username = "test Jira Id"; private final String accountUrl = "https://example.atlassian.net"; - private final List projectList = new ArrayList<>(); - private final List issueTypeList = new ArrayList<>(); - private final List statusList = new ArrayList<>(); + private final List spacesList = new ArrayList<>(); + private final List contentTypeList = new ArrayList<>(); - private ConfluenceSourceConfig createJiraSourceConfig(String authtype, boolean hasToken) throws Exception { + private ConfluenceSourceConfig createConfluenceSourceConfig(String authtype, boolean hasToken) throws Exception { PluginConfigVariable pcvAccessToken = null; PluginConfigVariable pcvRefreshToken = null; Map configMap = new HashMap<>(); @@ -70,30 +69,25 @@ private ConfluenceSourceConfig createJiraSourceConfig(String authtype, boolean h configMap.put("authentication", authenticationMap); - projectList.add("project1"); - projectList.add("project2"); + spacesList.add("space1"); + spacesList.add("space2"); - issueTypeList.add("issue type 1"); - issueTypeList.add("issue type 2"); - - statusList.add("status 1"); - statusList.add("status 2"); + contentTypeList.add("page"); + contentTypeList.add("blogpost"); Map filterMap = new HashMap<>(); Map projectMap = new HashMap<>(); Map issueTypeMap = new HashMap<>(); Map statusMap = new HashMap<>(); - issueTypeMap.put("include", issueTypeList); - filterMap.put("issue_type", issueTypeMap); + issueTypeMap.put("include", contentTypeList); + filterMap.put("page_type", issueTypeMap); - statusMap.put("include", statusList); - filterMap.put("status", statusMap); Map nameMap = new HashMap<>(); - nameMap.put("include", projectList); + nameMap.put("include", spacesList); projectMap.put("key", nameMap); - filterMap.put("project", projectMap); + filterMap.put("space", projectMap); configMap.put("filter", filterMap); @@ -111,9 +105,9 @@ private ConfluenceSourceConfig createJiraSourceConfig(String authtype, boolean h @Test void testGetters() throws Exception { - confluenceSourceConfig = createJiraSourceConfig(BASIC, false); - assertEquals(confluenceSourceConfig.getFilterConfig().getPageTypeConfig().getInclude(), issueTypeList); - assertEquals(confluenceSourceConfig.getFilterConfig().getSpaceConfig().getNameConfig().getInclude(), projectList); + confluenceSourceConfig = createConfluenceSourceConfig(BASIC, false); + assertEquals(confluenceSourceConfig.getFilterConfig().getPageTypeConfig().getInclude(), contentTypeList); + assertEquals(confluenceSourceConfig.getFilterConfig().getSpaceConfig().getNameConfig().getInclude(), spacesList); assertEquals(confluenceSourceConfig.getAccountUrl(), accountUrl); assertEquals(confluenceSourceConfig.getAuthenticationConfig().getBasicConfig().getPassword(), password); assertEquals(confluenceSourceConfig.getAuthenticationConfig().getBasicConfig().getUsername(), username); @@ -121,13 +115,13 @@ void testGetters() throws Exception { @Test void testFetchGivenOauthAttributeWrongAuthType() throws Exception { - confluenceSourceConfig = createJiraSourceConfig(BASIC, true); + confluenceSourceConfig = createConfluenceSourceConfig(BASIC, true); assertThrows(RuntimeException.class, () -> confluenceSourceConfig.getAuthenticationConfig().getOauth2Config().getAccessToken()); } @Test void testFetchGivenOauthAtrribute() throws Exception { - confluenceSourceConfig = createJiraSourceConfig(OAUTH2, true); + confluenceSourceConfig = createConfluenceSourceConfig(OAUTH2, true); assertEquals(accessToken, confluenceSourceConfig.getAuthenticationConfig().getOauth2Config().getAccessToken()); assertEquals(refreshToken, confluenceSourceConfig.getAuthenticationConfig().getOauth2Config().getRefreshToken()); assertEquals(clientId, confluenceSourceConfig.getAuthenticationConfig().getOauth2Config().getClientId()); diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceTest.java index ada00ab1d1..e6c62ee7fc 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceTest.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceTest.java @@ -21,9 +21,9 @@ import org.opensearch.dataprepper.model.plugin.PluginFactory; import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.model.source.coordinator.enhanced.EnhancedSourceCoordinator; -import org.opensearch.dataprepper.plugins.source.confluence.configuration.AuthenticationConfig; -import org.opensearch.dataprepper.plugins.source.confluence.configuration.BasicConfig; -import org.opensearch.dataprepper.plugins.source.confluence.rest.auth.ConfluenceAuthConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.configuration.AuthenticationConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.configuration.BasicConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.rest.auth.AtlassianAuthConfig; import org.opensearch.dataprepper.plugins.source.source_crawler.base.Crawler; import org.opensearch.dataprepper.plugins.source.source_crawler.base.PluginExecutorServiceProvider; @@ -35,7 +35,7 @@ import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import static org.opensearch.dataprepper.plugins.source.confluence.rest.auth.ConfluenceOauthConfig.ACCESSIBLE_RESOURCES; +import static org.opensearch.dataprepper.plugins.source.atlassian.rest.auth.AtlassianOauthConfig.ACCESSIBLE_RESOURCES; import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.BASIC; @ExtendWith(MockitoExtension.class) @@ -52,7 +52,7 @@ public class ConfluenceSourceTest { @Mock private ConfluenceSourceConfig confluenceSourceConfig; @Mock - private ConfluenceAuthConfig jiraOauthConfig; + private AtlassianAuthConfig jiraOauthConfig; @Mock private PluginFactory pluginFactory; @Mock diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceItemTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceItemTest.java index 1c22fb673a..827d0824a0 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceItemTest.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceItemTest.java @@ -15,9 +15,6 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; -import java.util.HashMap; -import java.util.Map; - import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; @@ -26,46 +23,40 @@ public class ConfluenceItemTest { - private ConfluenceItem issueBean; + private ConfluenceItem confluenceItemBean; @BeforeEach void setup() { - issueBean = new ConfluenceItem(); + confluenceItemBean = new ConfluenceItem(); } @Test public void testInitialization() { - assertNotNull(issueBean); + assertNotNull(confluenceItemBean); } @Test public void testNull() { - assertNull(issueBean.getId()); + assertNull(confluenceItemBean.getId()); } @Test void testNullCases() { - assertEquals(issueBean.getUpdatedTimeMillis(), 0); + assertEquals(confluenceItemBean.getUpdatedTimeMillis(), 0); } @Test void testGivenDateField() { - Map fieldsTestObject = new HashMap<>(); - fieldsTestObject.put("created", "2024-07-06T21:12:23.437-0700"); - fieldsTestObject.put("updated", "2022-07-06T21:12:23.106-0700"); - assertEquals(issueBean.getCreatedTimeMillis(), 1720325543000L); - assertEquals(issueBean.getUpdatedTimeMillis(), 1657167143000L); + assertEquals(confluenceItemBean.getCreatedTimeMillis(), 0L); + assertEquals(confluenceItemBean.getUpdatedTimeMillis(), 0L); } @Test public void testStringSettersAndGetters() { - String self = "selfTest"; - String key = "keyTest"; String id = "idTest"; - String expand = "expandTest"; - issueBean.setId(id); - assertEquals(issueBean.getId(), id); + confluenceItemBean.setId(id); + assertEquals(confluenceItemBean.getId(), id); } } diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClientTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClientTest.java index ea7f35350e..375e678183 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClientTest.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClientTest.java @@ -10,7 +10,6 @@ package org.opensearch.dataprepper.plugins.source.confluence.rest; -import com.fasterxml.jackson.core.JsonProcessingException; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.params.ParameterizedTest; @@ -44,8 +43,6 @@ import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.BASIC; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.OAUTH2; @ExtendWith(MockitoExtension.class) public class ConfluenceRestClientTest { @@ -71,15 +68,15 @@ private static Stream provideHttpStatusCodesWithExceptionClass() { } @ParameterizedTest - @ValueSource(strings = {"basic-auth-jira-pipeline.yaml"}) + @ValueSource(strings = {"basic-auth-confluence-pipeline.yaml"}) public void testFetchingJiraIssue(String configFileName) { - String exampleTicketResponse = "{\"id\":\"123\",\"key\":\"key\",\"self\":\"https://example.com/rest/api/2/issue/123\"}"; - doReturn(new ResponseEntity<>(exampleTicketResponse, HttpStatus.OK)).when(restTemplate).getForEntity(any(URI.class), any(Class.class)); - ConfluenceSourceConfig confluenceSourceConfig = ConfluenceServiceTest.createJiraConfigurationFromYaml(configFileName); + String examplePageResponse = "{\"id\":\"123\",\"key\":\"key\",\"self\":\"https://example.com/rest/api/2/issue/123\"}"; + doReturn(new ResponseEntity<>(examplePageResponse, HttpStatus.OK)).when(restTemplate).getForEntity(any(URI.class), any(Class.class)); + ConfluenceSourceConfig confluenceSourceConfig = ConfluenceServiceTest.createConfluenceConfigurationFromYaml(configFileName); AtlassianAuthConfig authConfig = new AtlassianAuthFactory(confluenceSourceConfig).getObject(); ConfluenceRestClient confluenceRestClient = new ConfluenceRestClient(restTemplate, authConfig, pluginMetrics); - String ticketDetails = confluenceRestClient.getContent("key"); - assertEquals(exampleTicketResponse, ticketDetails); + String pageDetails = confluenceRestClient.getContent("key"); + assertEquals(examplePageResponse, pageDetails); } @ParameterizedTest @@ -114,12 +111,9 @@ void testInvokeRestApiTokenExpiredInterruptException() throws InterruptedExcepti } @Test - public void testGetAllContentOauth2() throws JsonProcessingException { + public void testGetAllContentOauth2() { List issueType = new ArrayList<>(); - List issueStatus = new ArrayList<>(); - List projectKey = new ArrayList<>(); issueType.add("Task"); - ConfluenceSourceConfig confluenceSourceConfig = ConfluenceServiceTest.createJiraConfiguration(OAUTH2, issueType, issueStatus, projectKey); ConfluenceRestClient confluenceRestClient = new ConfluenceRestClient(restTemplate, authConfig, pluginMetrics); ConfluenceSearchResults mockConfluenceSearchResults = mock(ConfluenceSearchResults.class); doReturn("http://mock-service.jira.com/").when(authConfig).getUrl(); @@ -129,12 +123,9 @@ public void testGetAllContentOauth2() throws JsonProcessingException { } @Test - public void testGetAllContentBasic() throws JsonProcessingException { + public void testGetAllContentBasic() { List issueType = new ArrayList<>(); - List issueStatus = new ArrayList<>(); - List projectKey = new ArrayList<>(); issueType.add("Task"); - ConfluenceSourceConfig confluenceSourceConfig = ConfluenceServiceTest.createJiraConfiguration(BASIC, issueType, issueStatus, projectKey); ConfluenceRestClient confluenceRestClient = new ConfluenceRestClient(restTemplate, authConfig, pluginMetrics); ConfluenceSearchResults mockConfluenceSearchResults = mock(ConfluenceSearchResults.class); when(authConfig.getUrl()).thenReturn("https://example.com/"); @@ -144,7 +135,7 @@ public void testGetAllContentBasic() throws JsonProcessingException { } @Test - public void testRestApiAddressValidation() throws JsonProcessingException { + public void testRestApiAddressValidation() { when(authConfig.getUrl()).thenReturn("https://224.0.0.1/"); ConfluenceRestClient confluenceRestClient = new ConfluenceRestClient(restTemplate, authConfig, pluginMetrics); assertThrows(BadRequestException.class, () -> confluenceRestClient.getContent("TEST-1")); diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/CustomRestTemplateConfigTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/CustomRestTemplateConfigTest.java index 23297eb380..ea20598dff 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/CustomRestTemplateConfigTest.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/CustomRestTemplateConfigTest.java @@ -18,11 +18,14 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.dataprepper.model.plugin.PluginConfigVariable; +import org.opensearch.dataprepper.plugins.source.atlassian.configuration.AuthenticationConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.configuration.BasicConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.configuration.Oauth2Config; +import org.opensearch.dataprepper.plugins.source.atlassian.rest.BasicAuthInterceptor; +import org.opensearch.dataprepper.plugins.source.atlassian.rest.CustomRestTemplateConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.rest.OAuth2RequestInterceptor; +import org.opensearch.dataprepper.plugins.source.atlassian.rest.auth.AtlassianAuthConfig; import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceSourceConfig; -import org.opensearch.dataprepper.plugins.source.confluence.configuration.AuthenticationConfig; -import org.opensearch.dataprepper.plugins.source.confluence.configuration.BasicConfig; -import org.opensearch.dataprepper.plugins.source.confluence.configuration.Oauth2Config; -import org.opensearch.dataprepper.plugins.source.confluence.rest.auth.ConfluenceAuthConfig; import org.springframework.http.client.ClientHttpRequestInterceptor; import org.springframework.http.client.InterceptingClientHttpRequestFactory; import org.springframework.web.client.RestTemplate; @@ -47,7 +50,7 @@ class CustomRestTemplateConfigTest { private ConfluenceSourceConfig mockSourceConfig; @Mock - private ConfluenceAuthConfig mockAuthConfig; + private AtlassianAuthConfig mockAuthConfig; @Mock private BasicConfig mockBasicConfig; diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/resources/basic-auth-confluence-pipeline.yaml b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/resources/basic-auth-confluence-pipeline.yaml new file mode 100644 index 0000000000..0bfa6384e8 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/resources/basic-auth-confluence-pipeline.yaml @@ -0,0 +1,6 @@ +hosts: ["https://jira.com/"] +authentication: + basic: + username: "jiraId" + password: "jiraApiKey" + diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/resources/oauth2-auth-confluence-pipeline.yaml b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/resources/oauth2-auth-confluence-pipeline.yaml new file mode 100644 index 0000000000..7a4afd3abf --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/resources/oauth2-auth-confluence-pipeline.yaml @@ -0,0 +1,6 @@ +hosts: [ "https://jira.com/" ] +authentication: + oauth2: + client_id: "client_id" + client_secret: "client_secret" + From 4fde3e9ba768a16a87875ba70666b52ff6094e44 Mon Sep 17 00:00:00 2001 From: Santhosh Gandhe <1909520+san81@users.noreply.github.com> Date: Mon, 3 Feb 2025 16:55:24 -0800 Subject: [PATCH 09/14] atlassian commons gradle file Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com> --- .../atlassian-commons/build.gradle | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 data-prepper-plugins/saas-source-plugins/atlassian-commons/build.gradle diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/build.gradle b/data-prepper-plugins/saas-source-plugins/atlassian-commons/build.gradle new file mode 100644 index 0000000000..e8c2183214 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/build.gradle @@ -0,0 +1,29 @@ +plugins { + id 'java' +} + + +dependencies { + + implementation project(path: ':data-prepper-api') + implementation project(path: ':data-prepper-plugins:saas-source-plugins:source-crawler') + + implementation 'com.fasterxml.jackson.core:jackson-core' + implementation 'com.fasterxml.jackson.core:jackson-databind' + + implementation 'io.micrometer:micrometer-core' + implementation 'org.projectlombok:lombok:1.18.30' + annotationProcessor 'org.projectlombok:lombok:1.18.30' + implementation("org.springframework:spring-web:${libs.versions.spring.get()}") + + implementation(libs.spring.context) { + exclude group: 'commons-logging', module: 'commons-logging' + } + + testImplementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.4' + testImplementation project(path: ':data-prepper-test-common') +} + +test { + useJUnitPlatform() +} \ No newline at end of file From 6bd34511eb423d1bee54df1a1db0db9ada3f77a0 Mon Sep 17 00:00:00 2001 From: Santhosh Gandhe <1909520+san81@users.noreply.github.com> Date: Mon, 3 Feb 2025 17:15:00 -0800 Subject: [PATCH 10/14] moved the test to the atlassian commons Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com> --- .../atlassian-commons/build.gradle | 1 + .../rest/CustomRestTemplateConfigTest.java | 16 ++++++---------- .../confluence-source/build.gradle | 1 - .../source/confluence/ConfluenceService.java | 2 +- 4 files changed, 8 insertions(+), 12 deletions(-) rename data-prepper-plugins/saas-source-plugins/{confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence => atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian}/rest/CustomRestTemplateConfigTest.java (83%) diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/build.gradle b/data-prepper-plugins/saas-source-plugins/atlassian-commons/build.gradle index e8c2183214..fac3cd245b 100644 --- a/data-prepper-plugins/saas-source-plugins/atlassian-commons/build.gradle +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/build.gradle @@ -7,6 +7,7 @@ dependencies { implementation project(path: ':data-prepper-api') implementation project(path: ':data-prepper-plugins:saas-source-plugins:source-crawler') + implementation project(path: ':data-prepper-plugins:common') implementation 'com.fasterxml.jackson.core:jackson-core' implementation 'com.fasterxml.jackson.core:jackson-databind' diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/CustomRestTemplateConfigTest.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/CustomRestTemplateConfigTest.java similarity index 83% rename from data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/CustomRestTemplateConfigTest.java rename to data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/CustomRestTemplateConfigTest.java index ea20598dff..0e52d16eb6 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/CustomRestTemplateConfigTest.java +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/CustomRestTemplateConfigTest.java @@ -8,7 +8,7 @@ * */ -package org.opensearch.dataprepper.plugins.source.confluence.rest; +package org.opensearch.dataprepper.plugins.source.atlassian.rest; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.extension.ExtendWith; @@ -18,14 +18,12 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.dataprepper.model.plugin.PluginConfigVariable; +import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig; import org.opensearch.dataprepper.plugins.source.atlassian.configuration.AuthenticationConfig; import org.opensearch.dataprepper.plugins.source.atlassian.configuration.BasicConfig; import org.opensearch.dataprepper.plugins.source.atlassian.configuration.Oauth2Config; -import org.opensearch.dataprepper.plugins.source.atlassian.rest.BasicAuthInterceptor; -import org.opensearch.dataprepper.plugins.source.atlassian.rest.CustomRestTemplateConfig; -import org.opensearch.dataprepper.plugins.source.atlassian.rest.OAuth2RequestInterceptor; import org.opensearch.dataprepper.plugins.source.atlassian.rest.auth.AtlassianAuthConfig; -import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceSourceConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.utils.Constants; import org.springframework.http.client.ClientHttpRequestInterceptor; import org.springframework.http.client.InterceptingClientHttpRequestFactory; import org.springframework.web.client.RestTemplate; @@ -38,8 +36,6 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.when; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.BASIC; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.OAUTH2; @ExtendWith(MockitoExtension.class) class CustomRestTemplateConfigTest { @@ -47,7 +43,7 @@ class CustomRestTemplateConfigTest { private CustomRestTemplateConfig config; @Mock - private ConfluenceSourceConfig mockSourceConfig; + private AtlassianSourceConfig mockSourceConfig; @Mock private AtlassianAuthConfig mockAuthConfig; @@ -69,8 +65,8 @@ class CustomRestTemplateConfigTest { private static Stream provideAuthTypeAndExpectedInterceptorType() { return Stream.of( - Arguments.of(OAUTH2, OAuth2RequestInterceptor.class), - Arguments.of(BASIC, BasicAuthInterceptor.class), + Arguments.of(Constants.OAUTH2, OAuth2RequestInterceptor.class), + Arguments.of(Constants.BASIC, BasicAuthInterceptor.class), Arguments.of("Default", BasicAuthInterceptor.class), Arguments.of(null, BasicAuthInterceptor.class) ); diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/build.gradle b/data-prepper-plugins/saas-source-plugins/confluence-source/build.gradle index cd1cee26e8..07308a50f7 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/build.gradle +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/build.gradle @@ -23,7 +23,6 @@ dependencies { annotationProcessor 'org.projectlombok:lombok:1.18.30' testImplementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.4' - testImplementation project(path: ':data-prepper-plugins:saas-source-plugins:atlassian-commons') testImplementation project(path: ':data-prepper-test-common') implementation(libs.spring.context) { diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java index 5893e7065d..6e36bf3c1e 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java @@ -108,7 +108,7 @@ private void searchForNewContentAndAddToQueue(ConfluenceSourceConfig configurati addItemsToQueue(contentList, itemInfoQueue); } while (startAt < total); searchResultsFoundCounter.increment(total); - log.info("Number of tickets found in search api call: {}", total); + log.info("Number of content items found in search api call: {}", total); } /** From f8e8d456eaafd5441a015eb756449dac897ec4c5 Mon Sep 17 00:00:00 2001 From: Santhosh Gandhe <1909520+san81@users.noreply.github.com> Date: Mon, 3 Feb 2025 17:53:05 -0800 Subject: [PATCH 11/14] additional code coverage Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com> --- .../rest/AtlassianRestClientTest.java | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/AtlassianRestClientTest.java diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/AtlassianRestClientTest.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/AtlassianRestClientTest.java new file mode 100644 index 0000000000..7c2734f2c3 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/AtlassianRestClientTest.java @@ -0,0 +1,75 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.atlassian.rest; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.plugins.source.atlassian.rest.auth.AtlassianAuthConfig; +import org.opensearch.dataprepper.plugins.source.source_crawler.exception.UnAuthorizedException; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.util.UriComponentsBuilder; + +import java.net.URI; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +public class AtlassianRestClientTest { + + @Mock + private RestTemplate restTemplate; + + @Mock + private AtlassianAuthConfig authConfig; + + private static Stream provideHttpStatusCodesWithExceptionClass() { + return Stream.of( + Arguments.of(HttpStatus.FORBIDDEN, UnAuthorizedException.class), + Arguments.of(HttpStatus.UNAUTHORIZED, RuntimeException.class), + Arguments.of(HttpStatus.TOO_MANY_REQUESTS, RuntimeException.class), + Arguments.of(HttpStatus.INSUFFICIENT_STORAGE, RuntimeException.class) + ); + } + + @ParameterizedTest + @MethodSource("provideHttpStatusCodesWithExceptionClass") + void testInvokeRestApiTokenExpired(HttpStatus statusCode, Class expectedExceptionType) { + AtlassianRestClient atlassianRestClient = new AtlassianRestClient(restTemplate, authConfig); + atlassianRestClient.setSleepTimeMultiplier(1); + when(restTemplate.getForEntity(any(URI.class), any(Class.class))).thenThrow(new HttpClientErrorException(statusCode)); + URI uri = UriComponentsBuilder.fromHttpUrl("https://example.com/rest/api/2/issue/key").buildAndExpand().toUri(); + assertThrows(expectedExceptionType, () -> atlassianRestClient.invokeRestApi(uri, Object.class)); + } + + @Test + void testInvokeRestApiSuccessFullResponse() { + AtlassianRestClient atlassianRestClient = new AtlassianRestClient(restTemplate, authConfig); + atlassianRestClient.setSleepTimeMultiplier(1); + String apiReponse = "{\"api-response\":\"ok\"}"; + ResponseEntity responseEntity = new ResponseEntity(apiReponse, HttpStatus.OK); + when(restTemplate.getForEntity(any(URI.class), any(Class.class))).thenReturn(responseEntity); + URI uri = UriComponentsBuilder.fromHttpUrl("https://example.com/rest/api/2/issue/key").buildAndExpand().toUri(); + assertEquals(apiReponse, atlassianRestClient.invokeRestApi(uri, String.class).getBody()); + } + +} From 47ee2238f5d3af9488f62d3c2c52cbb4f76692ae Mon Sep 17 00:00:00 2001 From: Santhosh Gandhe <1909520+san81@users.noreply.github.com> Date: Mon, 3 Feb 2025 18:11:14 -0800 Subject: [PATCH 12/14] better names Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com> --- .../plugins/source/confluence/ConfluenceClient.java | 6 +++--- .../source/confluence/ConfluenceItemInfo.java | 4 ++-- .../source/confluence/ConfluenceIterator.java | 11 +++-------- .../confluence/models/ConfluenceSearchResults.java | 2 +- .../source/confluence/rest/ConfluenceRestClient.java | 12 ++++++------ .../confluence/utils/ConfluenceContentType.java | 8 ++++---- .../plugins/source/confluence/utils/Constants.java | 4 ---- .../source/confluence/utils/CqlConstants.java | 1 - .../confluence/utils/ConfluenceContentTypeTest.java | 12 ++++++------ 9 files changed, 25 insertions(+), 35 deletions(-) diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceClient.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceClient.java index b5ee7a1446..225a9721f6 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceClient.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceClient.java @@ -46,7 +46,7 @@ import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.SPACE; /** - * This class represents a Jira client. + * This class represents a Confluence client. */ @Named public class ConfluenceClient implements CrawlerClient { @@ -95,7 +95,7 @@ public void executePartition(SaasWorkerProgressState state, state.getKeyAttributes(), state.getItemIds().size()); List itemIds = state.getItemIds(); Map keyAttributes = state.getKeyAttributes(); - String project = (String) keyAttributes.get(SPACE); + String space = (String) keyAttributes.get(SPACE); Instant eventTime = state.getExportStartTime(); List itemInfos = new ArrayList<>(); for (String itemId : itemIds) { @@ -105,7 +105,7 @@ public void executePartition(SaasWorkerProgressState state, ItemInfo itemInfo = ConfluenceItemInfo.builder() .withItemId(itemId) .withId(itemId) - .withSpace(project) + .withSpace(space) .withEventTime(eventTime) .withMetadata(keyAttributes).build(); itemInfos.add(itemInfo); diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceItemInfo.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceItemInfo.java index f017f2588d..b1861b5b69 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceItemInfo.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceItemInfo.java @@ -144,11 +144,11 @@ public ConfluenceItemInfoBuilder withContentBean(ConfluenceItem contentItem) { contentItemMetadata.put(CREATED, contentItem.getCreatedTimeMillis()); contentItemMetadata.put(LAST_MODIFIED, contentItem.getUpdatedTimeMillis()); contentItemMetadata.put(CONTENT_ID, contentItem.getId()); - contentItemMetadata.put(ConfluenceService.CONTENT_TYPE, ConfluenceContentType.ISSUE.getType()); + contentItemMetadata.put(ConfluenceService.CONTENT_TYPE, ConfluenceContentType.PAGE.getType()); this.space = contentItem.getSpaceItem().getKey(); this.id = contentItem.getId(); - this.contentType = ConfluenceContentType.ISSUE.getType(); + this.contentType = ConfluenceContentType.PAGE.getType(); this.itemId = contentItem.getId(); this.metadata = contentItemMetadata; return this; diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceIterator.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceIterator.java index fb746c06ca..93ba62cec2 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceIterator.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceIterator.java @@ -103,11 +103,11 @@ public ItemInfo next() { /** * Initialize. * - * @param jiraChangeLogToken the jira change log token + * @param confluenceChangeLogToken the jira change log token */ - public void initialize(Instant jiraChangeLogToken) { + public void initialize(Instant confluenceChangeLogToken) { this.itemInfoQueue = new ConcurrentLinkedQueue<>(); - this.lastPollTime = jiraChangeLogToken; + this.lastPollTime = confluenceChangeLogToken; this.firstTime = true; } @@ -116,9 +116,4 @@ public List> showFutureList() { return futureList; } - @VisibleForTesting - public Queue showItemInfoQueue() { - return itemInfoQueue; - } - } diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceSearchResults.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceSearchResults.java index d56ca84305..fc55ad891a 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceSearchResults.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceSearchResults.java @@ -18,7 +18,7 @@ import java.util.List; /** - * The result of a JQL search. + * The result of a CQL search. */ @Getter @JsonIgnoreProperties(ignoreUnknown = true) diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClient.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClient.java index d3dcdad2d8..05367f20f9 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClient.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClient.java @@ -63,31 +63,31 @@ public ConfluenceRestClient(RestTemplate restTemplate, AtlassianAuthConfig authC } /** - * Method to get Issues. + * Method to get all Contents in a paginated fashion. * - * @param jql input parameter. + * @param cql input parameter. * @param startAt the start at * @return InputStream input stream */ @Timed(SEARCH_CALL_LATENCY_TIMER) - public ConfluenceSearchResults getAllContent(StringBuilder jql, int startAt) { + public ConfluenceSearchResults getAllContent(StringBuilder cql, int startAt) { String url = authConfig.getUrl() + REST_API_SEARCH; URI uri = UriComponentsBuilder.fromHttpUrl(url) .queryParam(MAX_RESULT, FIFTY) .queryParam(START_AT, startAt) - .queryParam(CQL_FIELD, jql) + .queryParam(CQL_FIELD, cql) .queryParam(EXPAND_FIELD, EXPAND_VALUE) .buildAndExpand().toUri(); return invokeRestApi(uri, ConfluenceSearchResults.class).getBody(); } /** - * Gets issue. + * Fetches content based on given the content id. * * @param contentId the item info - * @return the issue + * @return the content based on the given content id */ @Timed(PAGE_FETCH_LATENCY_TIMER) public String getContent(String contentId) { diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/ConfluenceContentType.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/ConfluenceContentType.java index 06ec12c652..462e1c9a4a 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/ConfluenceContentType.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/ConfluenceContentType.java @@ -15,11 +15,11 @@ @AllArgsConstructor public enum ConfluenceContentType { - PROJECT("PROJECT"), - ISSUE("ISSUE"), + SPACE("SPACE"), + PAGE("PAGE"), + BLOGPOST("BLOGPOST"), COMMENT("COMMENT"), - ATTACHMENT("ATTACHMENT"), - WORKLOG("WORKLOG"); + ATTACHMENT("ATTACHMENT"); @Getter private final String type; diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/Constants.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/Constants.java index e63d001a87..bb1460c51e 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/Constants.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/Constants.java @@ -15,10 +15,6 @@ */ public class Constants { - public static final int RETRY_ATTEMPT = 6; - - public static final String KEY = "key"; - public static final String NAME = "name"; public static final String SPACE = "space"; public static final String OAUTH2 = "OAuth2"; public static final String LAST_MODIFIED = "lastModified"; diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/CqlConstants.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/CqlConstants.java index 74b3a0f6c7..9d0de95966 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/CqlConstants.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/CqlConstants.java @@ -14,7 +14,6 @@ public class CqlConstants { public static final String GREATER_THAN_EQUALS = ">="; public static final String CLOSING_ROUND_BRACKET = ")"; - public static final String SLASH = "/"; public static final String SPACE_IN = " AND space in ("; public static final String SPACE_NOT_IN = " AND space not in ("; public static final String DELIMITER = "\",\""; diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/utils/ConfluenceContentTypeTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/utils/ConfluenceContentTypeTest.java index a17c617719..b81bd22b13 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/utils/ConfluenceContentTypeTest.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/utils/ConfluenceContentTypeTest.java @@ -18,19 +18,19 @@ public class ConfluenceContentTypeTest { @Test void testEnumConstants() { - assertNotNull(ConfluenceContentType.PROJECT); - assertNotNull(ConfluenceContentType.ISSUE); + assertNotNull(ConfluenceContentType.SPACE); + assertNotNull(ConfluenceContentType.PAGE); assertNotNull(ConfluenceContentType.COMMENT); assertNotNull(ConfluenceContentType.ATTACHMENT); - assertNotNull(ConfluenceContentType.WORKLOG); + assertNotNull(ConfluenceContentType.BLOGPOST); } @Test void testTypeValues() { - assertEquals("PROJECT", ConfluenceContentType.PROJECT.getType()); - assertEquals("ISSUE", ConfluenceContentType.ISSUE.getType()); + assertEquals("SPACE", ConfluenceContentType.SPACE.getType()); + assertEquals("PAGE", ConfluenceContentType.PAGE.getType()); assertEquals("COMMENT", ConfluenceContentType.COMMENT.getType()); assertEquals("ATTACHMENT", ConfluenceContentType.ATTACHMENT.getType()); - assertEquals("WORKLOG", ConfluenceContentType.WORKLOG.getType()); + assertEquals("BLOGPOST", ConfluenceContentType.BLOGPOST.getType()); } } From 15e0b4ae8f6738c55d3461e4e9477b61b92e65bd Mon Sep 17 00:00:00 2001 From: Santhosh Gandhe <1909520+san81@users.noreply.github.com> Date: Wed, 5 Feb 2025 15:02:23 -0800 Subject: [PATCH 13/14] better naming Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com> --- .../plugins/source/atlassian/rest/AtlassianRestClient.java | 6 +++--- .../source/atlassian/rest/auth/AtlassianOauthConfig.java | 6 +++--- .../plugins/source/atlassian/utils/Constants.java | 2 +- .../atlassian/rest/auth/AtlassianOauthConfigTest.java | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/AtlassianRestClient.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/AtlassianRestClient.java index 91c2ae633a..5e04ee7afa 100644 --- a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/AtlassianRestClient.java +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/AtlassianRestClient.java @@ -25,11 +25,11 @@ import java.util.List; import static org.opensearch.dataprepper.logging.DataPrepperMarkers.NOISY; -import static org.opensearch.dataprepper.plugins.source.atlassian.utils.Constants.RETRY_ATTEMPT; +import static org.opensearch.dataprepper.plugins.source.atlassian.utils.Constants.MAX_RETRIES; @Slf4j public class AtlassianRestClient { - + public static final List RETRY_ATTEMPT_SLEEP_TIME = List.of(1, 2, 5, 10, 20, 40); private int sleepTimeMultiplier = 1000; private final RestTemplate restTemplate; @@ -44,7 +44,7 @@ public AtlassianRestClient(RestTemplate restTemplate, AtlassianAuthConfig authCo protected ResponseEntity invokeRestApi(URI uri, Class responseType) throws BadRequestException { AddressValidation.validateInetAddress(AddressValidation.getInetAddress(uri.toString())); int retryCount = 0; - while (retryCount < RETRY_ATTEMPT) { + while (retryCount < MAX_RETRIES) { try { return restTemplate.getForEntity(uri, responseType); } catch (HttpClientErrorException ex) { diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianOauthConfig.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianOauthConfig.java index 654785c684..0c1197acbb 100644 --- a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianOauthConfig.java +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianOauthConfig.java @@ -30,7 +30,7 @@ import java.util.List; import java.util.Map; -import static org.opensearch.dataprepper.plugins.source.atlassian.utils.Constants.RETRY_ATTEMPT; +import static org.opensearch.dataprepper.plugins.source.atlassian.utils.Constants.MAX_RETRIES; import static org.opensearch.dataprepper.plugins.source.atlassian.utils.Constants.SLASH; /** @@ -83,7 +83,7 @@ public String getJiraAccountCloudId() { } int retryCount = 0; - while (retryCount < RETRY_ATTEMPT) { + while (retryCount < MAX_RETRIES) { retryCount++; try { HttpHeaders headers = new HttpHeaders(); @@ -103,7 +103,7 @@ public String getJiraAccountCloudId() { log.error("Error occurred while accessing resources: ", e); } } - throw new UnAuthorizedException(String.format("Access token expired. Unable to renew even after %s attempts", RETRY_ATTEMPT)); + throw new UnAuthorizedException(String.format("Access token expired. Unable to renew even after %s attempts", MAX_RETRIES)); } } diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/utils/Constants.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/utils/Constants.java index e4065d2c3f..17d702d201 100644 --- a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/utils/Constants.java +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/utils/Constants.java @@ -15,7 +15,7 @@ */ public class Constants { - public static final int RETRY_ATTEMPT = 6; + public static final int MAX_RETRIES = 6; public static final String OAUTH2 = "OAuth2"; public static final String BASIC = "Basic"; diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianOauthConfigTest.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianOauthConfigTest.java index 3131e5e502..49d4c129a7 100644 --- a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianOauthConfigTest.java +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianOauthConfigTest.java @@ -44,7 +44,7 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import static org.opensearch.dataprepper.plugins.source.atlassian.utils.Constants.RETRY_ATTEMPT; +import static org.opensearch.dataprepper.plugins.source.atlassian.utils.Constants.MAX_RETRIES; @ExtendWith(MockitoExtension.class) public class AtlassianOauthConfigTest { @@ -179,7 +179,7 @@ void testFailedToGetCloudId() { AtlassianOauthConfig jiraOauthConfig = new AtlassianOauthConfig(confluenceSourceConfig); jiraOauthConfig.restTemplate = restTemplateMock; assertThrows(RuntimeException.class, jiraOauthConfig::getUrl); - for (int i = 0; i <= RETRY_ATTEMPT; i++) { + for (int i = 0; i <= MAX_RETRIES; i++) { assertThrows(RuntimeException.class, jiraOauthConfig::getUrl); } } From 14a587026464d1fea74101714a735cb521491259 Mon Sep 17 00:00:00 2001 From: Santhosh Gandhe <1909520+san81@users.noreply.github.com> Date: Wed, 5 Feb 2025 15:05:00 -0800 Subject: [PATCH 14/14] making a default implementation Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com> --- .../source/atlassian/rest/auth/AtlassianAuthConfig.java | 3 ++- .../atlassian/rest/auth/AtlassianBasicAuthConfig.java | 6 +----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianAuthConfig.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianAuthConfig.java index 60b9529339..71da3d6dd2 100644 --- a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianAuthConfig.java +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianAuthConfig.java @@ -25,7 +25,8 @@ public interface AtlassianAuthConfig { /** * Initializes the credentials for the Jira instance. */ - void initCredentials(); + default void initCredentials() { + } /** * Renews the credentials for the Jira instance. diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianBasicAuthConfig.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianBasicAuthConfig.java index 68d81bd887..7b160ee65e 100644 --- a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianBasicAuthConfig.java +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianBasicAuthConfig.java @@ -30,11 +30,7 @@ public AtlassianBasicAuthConfig(AtlassianSourceConfig confluenceSourceConfig) { public String getUrl() { return accountUrl; } - - @Override - public void initCredentials() { - //do nothing for basic authentication - } + @Override public void renewCredentials() {