From 6f40d6592ce34cb371ceddb66160a26c43995e03 Mon Sep 17 00:00:00 2001 From: Santhosh Gandhe <1909520+san81@users.noreply.github.com> Date: Fri, 31 Jan 2025 19:24:32 -0800 Subject: [PATCH] Functional confluence version Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com> --- .../source/confluence/ConfluenceClient.java | 12 +- .../source/confluence/ConfluenceItemInfo.java | 68 ++++----- .../source/confluence/ConfluenceIterator.java | 2 +- .../source/confluence/ConfluenceService.java | 99 ++++++------- .../confluence/models/ConfluenceItem.java | 65 +++++++++ ...ults.java => ConfluenceSearchResults.java} | 23 +-- .../source/confluence/models/IssueBean.java | 125 ----------------- .../source/confluence/models/SpaceItem.java | 61 ++++++++ .../confluence/rest/ConfluenceRestClient.java | 63 ++++----- .../rest/auth/ConfluenceOauthConfig.java | 2 +- .../utils/ConfluenceConfigHelper.java | 21 ++- .../source/confluence/utils/Constants.java | 12 +- .../{JqlConstants.java => CqlConstants.java} | 16 +-- .../confluence/ConfluenceClientTest.java | 10 +- .../ConfluenceConfigHelperTest.java | 21 ++- .../confluence/ConfluenceItemInfoTest.java | 17 ++- .../confluence/ConfluenceIteratorTest.java | 113 +-------------- .../confluence/ConfluenceServiceTest.java | 67 ++++----- .../ConfluenceSourceConfigTest.java | 1 - .../confluence/models/ConfluenceItemTest.java | 71 ++++++++++ .../models/ConfluenceSearchResultsTest.java | 41 ++++++ .../confluence/models/IssueBeanTest.java | 131 ------------------ .../confluence/models/SearchResultsTest.java | 98 ------------- .../rest/ConfluenceRestClientTest.java | 41 +++--- .../utils/AddressValidationTest.java | 5 +- 25 files changed, 472 insertions(+), 713 deletions(-) create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceItem.java rename data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/{SearchResults.java => ConfluenceSearchResults.java} (53%) delete mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/IssueBean.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/SpaceItem.java rename data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/{JqlConstants.java => CqlConstants.java} (55%) create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceItemTest.java create mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceSearchResultsTest.java delete mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/IssueBeanTest.java delete mode 100644 data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/SearchResultsTest.java diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceClient.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceClient.java index 68874f8587..f4dc7cfe4f 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceClient.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceClient.java @@ -41,7 +41,7 @@ import java.util.stream.Collectors; import static java.util.concurrent.CompletableFuture.supplyAsync; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.PROJECT; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.SPACE; /** * This class represents a Jira client. @@ -93,7 +93,7 @@ public void executePartition(SaasWorkerProgressState state, state.getKeyAttributes(), state.getItemIds().size()); List itemIds = state.getItemIds(); Map keyAttributes = state.getKeyAttributes(); - String project = (String) keyAttributes.get(PROJECT); + String project = (String) keyAttributes.get(SPACE); Instant eventTime = state.getExportStartTime(); List itemInfos = new ArrayList<>(); for (String itemId : itemIds) { @@ -103,7 +103,7 @@ public void executePartition(SaasWorkerProgressState state, ItemInfo itemInfo = ConfluenceItemInfo.builder() .withItemId(itemId) .withId(itemId) - .withProject(project) + .withSpace(project) .withEventTime(eventTime) .withMetadata(keyAttributes).build(); itemInfos.add(itemInfo); @@ -112,12 +112,12 @@ public void executePartition(SaasWorkerProgressState state, String eventType = EventType.DOCUMENT.toString(); List> recordsToWrite = itemInfos .parallelStream() - .map(t -> (Supplier) (() -> service.getIssue(t.getId()))) + .map(t -> (Supplier) (() -> service.getContent(t.getId()))) .map(supplier -> supplyAsync(supplier, this.executorService)) .map(CompletableFuture::join) - .map(ticketJson -> { + .map(contentJson -> { try { - return objectMapper.readValue(ticketJson, new TypeReference<>() { + return objectMapper.readValue(contentJson, new TypeReference<>() { }); } catch (JsonProcessingException e) { throw new RuntimeException(e); diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceItemInfo.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceItemInfo.java index 15aa46009c..c6048caf65 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceItemInfo.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceItemInfo.java @@ -12,7 +12,7 @@ import lombok.Getter; import lombok.Setter; -import org.opensearch.dataprepper.plugins.source.confluence.models.IssueBean; +import org.opensearch.dataprepper.plugins.source.confluence.models.ConfluenceItem; import org.opensearch.dataprepper.plugins.source.confluence.utils.ConfluenceContentType; import org.opensearch.dataprepper.plugins.source.confluence.utils.Constants; import org.opensearch.dataprepper.plugins.source.source_crawler.model.ItemInfo; @@ -22,18 +22,18 @@ import java.util.Map; import java.util.UUID; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.CONTENT_ID; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.CONTENT_TITLE; import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.CREATED; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.ISSUE_KEY; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.PROJECT_KEY; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.PROJECT_NAME; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.UPDATED; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants._ISSUE; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.LAST_MODIFIED; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.SPACE_KEY; @Setter @Getter public class ConfluenceItemInfo implements ItemInfo { - private String project; - private String issueType; + private String space; + // either a page or a BlogPost + private String contentType; private String id; private String itemId; private Map metadata; @@ -41,14 +41,14 @@ public class ConfluenceItemInfo implements ItemInfo { public ConfluenceItemInfo(String id, String itemId, - String project, - String issueType, + String space, + String contentType, Map metadata, Instant eventTime ) { this.id = id; - this.project = project; - this.issueType = issueType; + this.space = space; + this.contentType = contentType; this.itemId = itemId; this.metadata = metadata; this.eventTime = eventTime; @@ -60,7 +60,7 @@ public static ConfluenceItemInfoBuilder builder() { @Override public String getPartitionKey() { - return project + "|" + issueType + "|" + UUID.randomUUID(); + return space + "|" + contentType + "|" + UUID.randomUUID(); } @Override @@ -70,12 +70,12 @@ public String getId() { @Override public Map getKeyAttributes() { - return Map.of(Constants.PROJECT, project); + return Map.of(Constants.SPACE, space); } @Override public Instant getLastModifiedAt() { - long updatedAtMillis = getMetadataField(Constants.UPDATED); + long updatedAtMillis = getMetadataField(Constants.LAST_MODIFIED); long createdAtMillis = getMetadataField(Constants.CREATED); return createdAtMillis > updatedAtMillis ? Instant.ofEpochMilli(createdAtMillis) : Instant.ofEpochMilli(updatedAtMillis); @@ -102,14 +102,14 @@ public static class ConfluenceItemInfoBuilder { private Instant eventTime; private String id; private String itemId; - private String project; - private String issueType; + private String space; + private String contentType; public ConfluenceItemInfoBuilder() { } public ConfluenceItemInfo build() { - return new ConfluenceItemInfo(id, itemId, project, issueType, metadata, eventTime); + return new ConfluenceItemInfo(id, itemId, space, contentType, metadata, eventTime); } public ConfluenceItemInfoBuilder withMetadata(Map metadata) { @@ -132,25 +132,25 @@ public ConfluenceItemInfoBuilder withId(String id) { return this; } - public ConfluenceItemInfoBuilder withProject(String project) { - this.project = project; + public ConfluenceItemInfoBuilder withSpace(String space) { + this.space = space; return this; } - public ConfluenceItemInfoBuilder withIssueBean(IssueBean issue) { - Map issueMetadata = new HashMap<>(); - issueMetadata.put(PROJECT_KEY, issue.getProject()); - issueMetadata.put(PROJECT_NAME, issue.getProjectName()); - issueMetadata.put(CREATED, issue.getCreatedTimeMillis()); - issueMetadata.put(UPDATED, issue.getUpdatedTimeMillis()); - issueMetadata.put(ISSUE_KEY, issue.getKey()); - issueMetadata.put(ConfluenceService.CONTENT_TYPE, ConfluenceContentType.ISSUE.getType()); - - this.project = issue.getProject(); - this.id = issue.getKey(); - this.issueType = ConfluenceContentType.ISSUE.getType(); - this.itemId = _ISSUE + issueMetadata.get(PROJECT_KEY) + "-" + issue.getKey(); - this.metadata = issueMetadata; + public ConfluenceItemInfoBuilder withIssueBean(ConfluenceItem contentItem) { + Map contentItemMetadata = new HashMap<>(); + contentItemMetadata.put(SPACE_KEY, contentItem.getSpaceItem().getKey()); + contentItemMetadata.put(CONTENT_TITLE, contentItem.getTitle()); + contentItemMetadata.put(CREATED, contentItem.getCreatedTimeMillis()); + contentItemMetadata.put(LAST_MODIFIED, contentItem.getUpdatedTimeMillis()); + contentItemMetadata.put(CONTENT_ID, contentItem.getId()); + contentItemMetadata.put(ConfluenceService.CONTENT_TYPE, ConfluenceContentType.ISSUE.getType()); + + this.space = contentItem.getSpaceItem().getKey(); + this.id = contentItem.getId(); + this.contentType = ConfluenceContentType.ISSUE.getType(); + this.itemId = contentItem.getId(); + this.metadata = contentItemMetadata; return this; } } diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceIterator.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceIterator.java index bc67568083..fb746c06ca 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceIterator.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceIterator.java @@ -88,7 +88,7 @@ private boolean isCrawlerRunning() { private void startCrawlerThreads() { futureList.add(crawlerTaskExecutor.submit(() -> - service.getJiraEntities(sourceConfig, lastPollTime, itemInfoQueue), false)); + service.getPages(sourceConfig, lastPollTime, itemInfoQueue), false)); } @Override diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java index 81ddd8ce12..457c98c40c 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java @@ -14,8 +14,8 @@ import lombok.extern.slf4j.Slf4j; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.plugins.source.confluence.exception.BadRequestException; -import org.opensearch.dataprepper.plugins.source.confluence.models.IssueBean; -import org.opensearch.dataprepper.plugins.source.confluence.models.SearchResults; +import org.opensearch.dataprepper.plugins.source.confluence.models.ConfluenceItem; +import org.opensearch.dataprepper.plugins.source.confluence.models.ConfluenceSearchResults; import org.opensearch.dataprepper.plugins.source.confluence.rest.ConfluenceRestClient; import org.opensearch.dataprepper.plugins.source.confluence.utils.ConfluenceConfigHelper; import org.opensearch.dataprepper.plugins.source.source_crawler.model.ItemInfo; @@ -23,6 +23,9 @@ import javax.inject.Named; import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -32,16 +35,16 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.UPDATED; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.CLOSING_ROUND_BRACKET; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.DELIMITER; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.GREATER_THAN_EQUALS; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.ISSUE_TYPE_IN; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.ISSUE_TYPE_NOT_IN; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.PREFIX; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.PROJECT_IN; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.PROJECT_NOT_IN; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.SUFFIX; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.LAST_MODIFIED; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.CqlConstants.CLOSING_ROUND_BRACKET; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.CqlConstants.CONTENT_TYPE_IN; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.CqlConstants.CONTENT_TYPE_NOT_IN; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.CqlConstants.DELIMITER; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.CqlConstants.GREATER_THAN_EQUALS; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.CqlConstants.PREFIX; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.CqlConstants.SPACE_IN; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.CqlConstants.SPACE_NOT_IN; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.CqlConstants.SUFFIX; /** @@ -74,14 +77,14 @@ public ConfluenceService(ConfluenceSourceConfig confluenceSourceConfig, Confluen * @param configuration the configuration. * @param timestamp timestamp. */ - public void getJiraEntities(ConfluenceSourceConfig configuration, Instant timestamp, Queue itemInfoQueue) { + public void getPages(ConfluenceSourceConfig configuration, Instant timestamp, Queue itemInfoQueue) { log.trace("Started to fetch entities"); - searchForNewTicketsAndAddToQueue(configuration, timestamp, itemInfoQueue); + searchForNewContentAndAddToQueue(configuration, timestamp, itemInfoQueue); log.trace("Creating item information and adding in queue"); } - public String getIssue(String issueKey) { - return confluenceRestClient.getIssue(issueKey); + public String getContent(String contentId) { + return confluenceRestClient.getContent(contentId); } /** @@ -90,17 +93,17 @@ public String getIssue(String issueKey) { * @param configuration Input Parameter * @param timestamp Input Parameter */ - private void searchForNewTicketsAndAddToQueue(ConfluenceSourceConfig configuration, Instant timestamp, + private void searchForNewContentAndAddToQueue(ConfluenceSourceConfig configuration, Instant timestamp, Queue itemInfoQueue) { log.trace("Looking for Add/Modified tickets with a Search API call"); - StringBuilder jql = createIssueFilterCriteria(configuration, timestamp); + StringBuilder cql = createContentFilterCriteria(configuration, timestamp); int total; int startAt = 0; do { - SearchResults searchIssues = confluenceRestClient.getAllIssues(jql, startAt, configuration); - List issueList = new ArrayList<>(searchIssues.getIssues()); - total = searchIssues.getTotal(); - startAt += searchIssues.getIssues().size(); + ConfluenceSearchResults searchIssues = confluenceRestClient.getAllContent(cql, startAt); + List issueList = new ArrayList<>(searchIssues.getResults()); + total = searchIssues.getSize(); + startAt += searchIssues.getResults().size(); addItemsToQueue(issueList, itemInfoQueue); } while (startAt < total); searchResultsFoundCounter.increment(total); @@ -113,7 +116,7 @@ private void searchForNewTicketsAndAddToQueue(ConfluenceSourceConfig configurati * @param issueList Issue list. * @param itemInfoQueue Item info queue. */ - private void addItemsToQueue(List issueList, Queue itemInfoQueue) { + private void addItemsToQueue(List issueList, Queue itemInfoQueue) { issueList.forEach(issue -> { itemInfoQueue.add(ConfluenceItemInfo.builder().withEventTime(Instant.now()).withIssueBean(issue).build()); }); @@ -121,63 +124,65 @@ private void addItemsToQueue(List issueList, Queue itemInfo /** - * Method for creating Issue Filter Criteria. + * Method for creating Content Filter Criteria. * * @param configuration Input Parameter * @param ts Input Parameter * @return String Builder */ - private StringBuilder createIssueFilterCriteria(ConfluenceSourceConfig configuration, Instant ts) { + private StringBuilder createContentFilterCriteria(ConfluenceSourceConfig configuration, Instant ts) { - log.info("Creating issue filter criteria"); - if (!CollectionUtils.isEmpty(ConfluenceConfigHelper.getProjectNameIncludeFilter(configuration)) || !CollectionUtils.isEmpty(ConfluenceConfigHelper.getProjectNameExcludeFilter(configuration))) { - validateProjectFilters(configuration); + log.info("Creating content filter criteria"); + if (!CollectionUtils.isEmpty(ConfluenceConfigHelper.getSpacesNameIncludeFilter(configuration)) || !CollectionUtils.isEmpty(ConfluenceConfigHelper.getSpacesNameExcludeFilter(configuration))) { + validateSpaceFilters(configuration); } - StringBuilder jiraQl = new StringBuilder(UPDATED + GREATER_THAN_EQUALS + ts.toEpochMilli()); - if (!CollectionUtils.isEmpty(ConfluenceConfigHelper.getProjectNameIncludeFilter(configuration))) { - jiraQl.append(PROJECT_IN).append(ConfluenceConfigHelper.getProjectNameIncludeFilter(configuration).stream() + String formattedTimeStamp = LocalDateTime.ofInstant(ts, ZoneId.systemDefault()) + .format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm")); + StringBuilder cQl = new StringBuilder(LAST_MODIFIED + GREATER_THAN_EQUALS + "\"" + formattedTimeStamp + "\""); + if (!CollectionUtils.isEmpty(ConfluenceConfigHelper.getSpacesNameIncludeFilter(configuration))) { + cQl.append(SPACE_IN).append(ConfluenceConfigHelper.getSpacesNameIncludeFilter(configuration).stream() .collect(Collectors.joining(DELIMITER, PREFIX, SUFFIX))) .append(CLOSING_ROUND_BRACKET); } - if (!CollectionUtils.isEmpty(ConfluenceConfigHelper.getProjectNameExcludeFilter(configuration))) { - jiraQl.append(PROJECT_NOT_IN).append(ConfluenceConfigHelper.getProjectNameExcludeFilter(configuration).stream() + if (!CollectionUtils.isEmpty(ConfluenceConfigHelper.getSpacesNameExcludeFilter(configuration))) { + cQl.append(SPACE_NOT_IN).append(ConfluenceConfigHelper.getSpacesNameExcludeFilter(configuration).stream() .collect(Collectors.joining(DELIMITER, PREFIX, SUFFIX))) .append(CLOSING_ROUND_BRACKET); } - if (!CollectionUtils.isEmpty(ConfluenceConfigHelper.getIssueTypeIncludeFilter(configuration))) { - jiraQl.append(ISSUE_TYPE_IN).append(ConfluenceConfigHelper.getIssueTypeIncludeFilter(configuration).stream() + if (!CollectionUtils.isEmpty(ConfluenceConfigHelper.getContentTypeIncludeFilter(configuration))) { + cQl.append(CONTENT_TYPE_IN).append(ConfluenceConfigHelper.getContentTypeIncludeFilter(configuration).stream() .collect(Collectors.joining(DELIMITER, PREFIX, SUFFIX))) .append(CLOSING_ROUND_BRACKET); } - if (!CollectionUtils.isEmpty(ConfluenceConfigHelper.getIssueTypeExcludeFilter(configuration))) { - jiraQl.append(ISSUE_TYPE_NOT_IN).append(ConfluenceConfigHelper.getIssueTypeExcludeFilter(configuration).stream() + if (!CollectionUtils.isEmpty(ConfluenceConfigHelper.getContentTypeExcludeFilter(configuration))) { + cQl.append(CONTENT_TYPE_NOT_IN).append(ConfluenceConfigHelper.getContentTypeExcludeFilter(configuration).stream() .collect(Collectors.joining(DELIMITER, PREFIX, SUFFIX))) .append(CLOSING_ROUND_BRACKET); } - log.error("Created issue filter criteria JiraQl query: {}", jiraQl); - return jiraQl; + log.error("Created issue filter criteria JiraQl query: {}", cQl); + return cQl; } /** - * Method for Validating Project Filters. + * Method for Validating Space Filters. * * @param configuration Input Parameter */ - private void validateProjectFilters(ConfluenceSourceConfig configuration) { + private void validateSpaceFilters(ConfluenceSourceConfig configuration) { log.trace("Validating project filters"); List badFilters = new ArrayList<>(); Set includedProjects = new HashSet<>(); List includedAndExcludedProjects = new ArrayList<>(); Pattern regex = Pattern.compile("[^A-Z0-9]"); - ConfluenceConfigHelper.getProjectNameIncludeFilter(configuration).forEach(projectFilter -> { + ConfluenceConfigHelper.getSpacesNameIncludeFilter(configuration).forEach(projectFilter -> { Matcher matcher = regex.matcher(projectFilter); includedProjects.add(projectFilter); if (matcher.find() || projectFilter.length() <= 1 || projectFilter.length() > 10) { badFilters.add(projectFilter); } }); - ConfluenceConfigHelper.getProjectNameExcludeFilter(configuration).forEach(projectFilter -> { + ConfluenceConfigHelper.getSpacesNameExcludeFilter(configuration).forEach(projectFilter -> { Matcher matcher = regex.matcher(projectFilter); if (includedProjects.contains(projectFilter)) { includedAndExcludedProjects.add(projectFilter); @@ -188,16 +193,16 @@ private void validateProjectFilters(ConfluenceSourceConfig configuration) { }); if (!badFilters.isEmpty()) { String filters = String.join("\"" + badFilters + "\"", ", "); - log.error("One or more invalid project keys found in filter configuration: {}", badFilters); + log.error("One or more invalid Space keys found in filter configuration: {}", badFilters); throw new BadRequestException("Bad request exception occurred " + - "Invalid project key found in filter configuration for " + "Invalid Space key found in filter configuration for " + filters); } if (!includedAndExcludedProjects.isEmpty()) { String filters = String.join("\"" + includedAndExcludedProjects + "\"", ", "); - log.error("One or more project keys found in both include and exclude: {}", includedAndExcludedProjects); + log.error("One or more Space keys found in both include and exclude: {}", includedAndExcludedProjects); throw new BadRequestException("Bad request exception occurred " + - "Project filters is invalid because the following projects are listed in both include and exclude" + "Space filters is invalid because the following space are listed in both include and exclude" + filters); } diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceItem.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceItem.java new file mode 100644 index 0000000000..1806627793 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceItem.java @@ -0,0 +1,65 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.models; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Getter; +import lombok.Setter; + + +@Setter +@Getter +@JsonIgnoreProperties(ignoreUnknown = true) +public class ConfluenceItem { + + /** + * The ID of the issue. + */ + @JsonProperty("id") + private String id = null; + + /** + * The type of the issue. + */ + @JsonProperty("type") + private String type = null; + + /** + * The type of the issue. + */ + @JsonProperty("status") + private String status = null; + + /** + * The type of the issue. + */ + @JsonProperty("title") + private String title = null; + + /** + * Space this content belongs to + */ + @JsonProperty("space") + private SpaceItem spaceItem; + + @JsonIgnore + public long getCreatedTimeMillis() { + return 0L; + } + + @JsonIgnore + public long getUpdatedTimeMillis() { + return 0L; + } + +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/SearchResults.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceSearchResults.java similarity index 53% rename from data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/SearchResults.java rename to data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceSearchResults.java index 17fd06803c..d56ca84305 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/SearchResults.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceSearchResults.java @@ -11,6 +11,7 @@ package org.opensearch.dataprepper.plugins.source.confluence.models; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Getter; @@ -20,20 +21,20 @@ * The result of a JQL search. */ @Getter -public class SearchResults { - @JsonProperty("expand") - private String expand = null; +@JsonIgnoreProperties(ignoreUnknown = true) +public class ConfluenceSearchResults { - @JsonProperty("startAt") - private Integer startAt = null; - @JsonProperty("maxResults") - private Integer maxResults = null; + @JsonProperty("start") + private Integer start = null; - @JsonProperty("total") - private Integer total = null; + @JsonProperty("limit") + private Integer limit = null; - @JsonProperty("issues") - private List issues = null; + @JsonProperty("size") + private Integer size = null; + + @JsonProperty("results") + private List results = null; } diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/IssueBean.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/IssueBean.java deleted file mode 100644 index 57b6f49774..0000000000 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/IssueBean.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - */ - -package org.opensearch.dataprepper.plugins.source.confluence.models; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Getter; -import lombok.Setter; - -import java.time.OffsetDateTime; -import java.time.format.DateTimeFormatter; -import java.util.Date; -import java.util.Map; -import java.util.Objects; -import java.util.regex.Pattern; - -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.CREATED; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.KEY; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.NAME; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.PROJECT; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.UPDATED; - - -public class IssueBean { - - /** - * Expand options that include additional issue details in the response. - */ - @Getter - @Setter - @JsonProperty("expand") - private String expand = null; - /** - * The ID of the issue. - */ - @Getter - @Setter - @JsonProperty("id") - private String id = null; - /** - * The URL of the issue details. - */ - @Getter - @Setter - @JsonProperty("self") - private String self = null; - /** - * The key of the issue. - */ - @Getter - @Setter - @JsonProperty("key") - private String key = null; - @Getter - @Setter - @JsonProperty("renderedFields") - private Map renderedFields = null; - @Getter - @Setter - @JsonProperty("properties") - private Map properties = null; - @Getter - @Setter - @JsonProperty("names") - private Map names = null; - @Getter - @Setter - @JsonProperty("fields") - private Map fields = null; - @JsonIgnore - private final Pattern JiraDateTimePattern = Pattern.compile( - "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}[-+]\\d{4}$"); - @JsonIgnore - private final DateTimeFormatter offsetDateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ"); - - @JsonIgnore - public String getProject() { - if (fields != null && Objects.nonNull(((Map) fields.get(PROJECT)).get(KEY))) { - return ((Map) fields.get(PROJECT)).get(KEY).toString(); - } - return null; - } - - @JsonIgnore - public String getProjectName() { - if (fields != null && Objects.nonNull(((Map) fields.get(PROJECT)).get(NAME))) { - return ((Map) fields.get(PROJECT)).get(NAME).toString(); - } - return null; - } - - @JsonIgnore - public long getCreatedTimeMillis() { - return getGivenDateField(CREATED); - } - - @JsonIgnore - public long getUpdatedTimeMillis() { - return getGivenDateField(UPDATED); - } - - @JsonIgnore - private long getGivenDateField(String dateTimeFieldToPull) { - long dateTimeField = 0; - - if (fields != null && Objects.nonNull(fields.get(dateTimeFieldToPull)) && JiraDateTimePattern.matcher(fields.get(dateTimeFieldToPull) - .toString()).matches()) { - String charSequence = fields.get(dateTimeFieldToPull).toString(); - OffsetDateTime offsetDateTime = OffsetDateTime.parse(charSequence, offsetDateTimeFormatter); - new Date(offsetDateTime.toInstant().toEpochMilli()); - dateTimeField = offsetDateTime.toEpochSecond() * 1000; - } - return dateTimeField; - } - - -} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/SpaceItem.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/SpaceItem.java new file mode 100644 index 0000000000..9e8733b582 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/SpaceItem.java @@ -0,0 +1,61 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.models; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Getter; +import lombok.Setter; + + +@Setter +@Getter +@JsonIgnoreProperties(ignoreUnknown = true) +public class SpaceItem { + + /** + * The ID of the issue. + */ + @JsonProperty("id") + private int id; + + /** + * The type of the issue. + */ + @JsonProperty("key") + private String key = null; + + /** + * The type of the issue. + */ + @JsonProperty("alias") + private String alias = null; + + /** + * The type of the issue. + */ + @JsonProperty("name") + private String name = null; + + /** + * The type of the issue. + */ + @JsonProperty("status") + private String status = null; + + /** + * The type of the issue. + */ + @JsonProperty("type") + private String type = null; + + +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClient.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClient.java index 9152b88a06..f0bf79be14 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClient.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClient.java @@ -16,10 +16,9 @@ import io.micrometer.core.instrument.Timer; import lombok.extern.slf4j.Slf4j; import org.opensearch.dataprepper.metrics.PluginMetrics; -import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceSourceConfig; import org.opensearch.dataprepper.plugins.source.confluence.exception.BadRequestException; import org.opensearch.dataprepper.plugins.source.confluence.exception.UnAuthorizedException; -import org.opensearch.dataprepper.plugins.source.confluence.models.SearchResults; +import org.opensearch.dataprepper.plugins.source.confluence.models.ConfluenceSearchResults; import org.opensearch.dataprepper.plugins.source.confluence.rest.auth.ConfluenceAuthConfig; import org.opensearch.dataprepper.plugins.source.confluence.utils.AddressValidation; import org.springframework.http.HttpStatus; @@ -34,78 +33,76 @@ import static org.opensearch.dataprepper.logging.DataPrepperMarkers.NOISY; import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.RETRY_ATTEMPT; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.EXPAND_FIELD; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.EXPAND_VALUE; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.JQL_FIELD; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.CqlConstants.CQL_FIELD; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.CqlConstants.EXPAND_FIELD; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.CqlConstants.EXPAND_VALUE; @Slf4j @Named public class ConfluenceRestClient { - public static final String REST_API_SEARCH = "rest/api/3/search"; - public static final String REST_API_FETCH_ISSUE = "rest/api/3/issue"; - public static final String REST_API_PROJECTS = "/rest/api/3/project/search"; + public static final String REST_API_SEARCH = "wiki/rest/api/content/search"; + public static final String REST_API_FETCH_CONTENT = "wiki/rest/api/content/"; + public static final String REST_API_CONTENT_EXPAND_PARAM = "?expand=body.view"; + //public static final String REST_API_SPACES = "/rest/api/api/spaces"; public static final String FIFTY = "50"; public static final String START_AT = "startAt"; public static final String MAX_RESULT = "maxResults"; public static final List RETRY_ATTEMPT_SLEEP_TIME = List.of(1, 2, 5, 10, 20, 40); - private static final String TICKET_FETCH_LATENCY_TIMER = "ticketFetchLatency"; + private static final String PAGE_FETCH_LATENCY_TIMER = "pageFetchLatency"; private static final String SEARCH_CALL_LATENCY_TIMER = "searchCallLatency"; - private static final String PROJECTS_FETCH_LATENCY_TIMER = "projectFetchLatency"; - private static final String ISSUES_REQUESTED = "issuesRequested"; + private static final String SPACES_FETCH_LATENCY_TIMER = "spacesFetchLatency"; + private static final String PAGES_REQUESTED = "pagesRequested"; private int sleepTimeMultiplier = 1000; private final RestTemplate restTemplate; private final ConfluenceAuthConfig authConfig; - private final Timer ticketFetchLatencyTimer; + private final Timer contentFetchLatencyTimer; private final Timer searchCallLatencyTimer; - private final Timer projectFetchLatencyTimer; - private final Counter issuesRequestedCounter; - private final PluginMetrics jiraPluginMetrics = PluginMetrics.fromNames("jiraRestClient", "aws"); + private final Timer spaceFetchLatencyTimer; + private final Counter contentRequestedCounter; - public ConfluenceRestClient(RestTemplate restTemplate, ConfluenceAuthConfig authConfig) { + public ConfluenceRestClient(RestTemplate restTemplate, ConfluenceAuthConfig authConfig, + PluginMetrics pluginMetrics) { this.restTemplate = restTemplate; this.authConfig = authConfig; - ticketFetchLatencyTimer = jiraPluginMetrics.timer(TICKET_FETCH_LATENCY_TIMER); - searchCallLatencyTimer = jiraPluginMetrics.timer(SEARCH_CALL_LATENCY_TIMER); - projectFetchLatencyTimer = jiraPluginMetrics.timer(PROJECTS_FETCH_LATENCY_TIMER); - - issuesRequestedCounter = jiraPluginMetrics.counter(ISSUES_REQUESTED); + contentFetchLatencyTimer = pluginMetrics.timer(PAGE_FETCH_LATENCY_TIMER); + searchCallLatencyTimer = pluginMetrics.timer(SEARCH_CALL_LATENCY_TIMER); + spaceFetchLatencyTimer = pluginMetrics.timer(SPACES_FETCH_LATENCY_TIMER); + contentRequestedCounter = pluginMetrics.counter(PAGES_REQUESTED); } /** * Method to get Issues. * - * @param jql input parameter. - * @param startAt the start at - * @param configuration input parameter. + * @param jql input parameter. + * @param startAt the start at * @return InputStream input stream */ @Timed(SEARCH_CALL_LATENCY_TIMER) - public SearchResults getAllIssues(StringBuilder jql, int startAt, - ConfluenceSourceConfig configuration) { + public ConfluenceSearchResults getAllContent(StringBuilder jql, int startAt) { String url = authConfig.getUrl() + REST_API_SEARCH; URI uri = UriComponentsBuilder.fromHttpUrl(url) .queryParam(MAX_RESULT, FIFTY) .queryParam(START_AT, startAt) - .queryParam(JQL_FIELD, jql) + .queryParam(CQL_FIELD, jql) .queryParam(EXPAND_FIELD, EXPAND_VALUE) .buildAndExpand().toUri(); - return invokeRestApi(uri, SearchResults.class).getBody(); + return invokeRestApi(uri, ConfluenceSearchResults.class).getBody(); } /** * Gets issue. * - * @param issueKey the item info + * @param contentId the item info * @return the issue */ - @Timed(TICKET_FETCH_LATENCY_TIMER) - public String getIssue(String issueKey) { - issuesRequestedCounter.increment(); - String url = authConfig.getUrl() + REST_API_FETCH_ISSUE + "/" + issueKey; + @Timed(PAGE_FETCH_LATENCY_TIMER) + public String getContent(String contentId) { + contentRequestedCounter.increment(); + String url = authConfig.getUrl() + REST_API_FETCH_CONTENT + "/" + contentId + REST_API_CONTENT_EXPAND_PARAM; URI uri = UriComponentsBuilder.fromHttpUrl(url).buildAndExpand().toUri(); return invokeRestApi(uri, String.class).getBody(); } diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceOauthConfig.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceOauthConfig.java index 51638b4dcb..dea42f2917 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceOauthConfig.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/auth/ConfluenceOauthConfig.java @@ -31,7 +31,7 @@ import java.util.Map; import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.RETRY_ATTEMPT; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants.SLASH; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.CqlConstants.SLASH; /** * The type Jira service. diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/ConfluenceConfigHelper.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/ConfluenceConfigHelper.java index 09e334eea0..9f36bfe91d 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/ConfluenceConfigHelper.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/ConfluenceConfigHelper.java @@ -21,24 +21,24 @@ import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.OAUTH2; /** - * The type Jira configuration. + * The type Confluence configuration. */ @Slf4j public class ConfluenceConfigHelper { /** - * Get Issue Types Filter from repository configuration. + * Get Content Types Filter from configuration. * - * @return List Issue Type Filter. + * @return List Content Type Filter. */ - public static List getIssueTypeIncludeFilter(ConfluenceSourceConfig repositoryConfiguration) { + public static List getContentTypeIncludeFilter(ConfluenceSourceConfig repositoryConfiguration) { if (repositoryConfiguration.getFilterConfig() == null || repositoryConfiguration.getFilterConfig().getPageTypeConfig() == null) { return new ArrayList<>(); } return repositoryConfiguration.getFilterConfig().getPageTypeConfig().getInclude(); } - public static List getIssueTypeExcludeFilter(ConfluenceSourceConfig repositoryConfiguration) { + public static List getContentTypeExcludeFilter(ConfluenceSourceConfig repositoryConfiguration) { if (repositoryConfiguration.getFilterConfig() == null || repositoryConfiguration.getFilterConfig().getPageTypeConfig() == null) { return new ArrayList<>(); } @@ -46,12 +46,11 @@ public static List getIssueTypeExcludeFilter(ConfluenceSourceConfig repo } /** - * Get Project Filter Types from repository configuration. - * public static final String ST = "status"; + * Get Space Filter Types from configuration. * - * @return List Project Filter. + * @return List Space Filter. */ - public static List getProjectNameIncludeFilter(ConfluenceSourceConfig repositoryConfiguration) { + public static List getSpacesNameIncludeFilter(ConfluenceSourceConfig repositoryConfiguration) { if (repositoryConfiguration.getFilterConfig() == null || repositoryConfiguration.getFilterConfig().getSpaceConfig() == null || repositoryConfiguration.getFilterConfig().getSpaceConfig().getNameConfig() == null) { @@ -60,7 +59,7 @@ public static List getProjectNameIncludeFilter(ConfluenceSourceConfig re return repositoryConfiguration.getFilterConfig().getSpaceConfig().getNameConfig().getInclude(); } - public static List getProjectNameExcludeFilter(ConfluenceSourceConfig repositoryConfiguration) { + public static List getSpacesNameExcludeFilter(ConfluenceSourceConfig repositoryConfiguration) { if (repositoryConfiguration.getFilterConfig() == null || repositoryConfiguration.getFilterConfig().getSpaceConfig() == null || repositoryConfiguration.getFilterConfig().getSpaceConfig().getNameConfig() == null) { @@ -85,7 +84,7 @@ public static boolean validateConfig(ConfluenceSourceConfig config) { if (BASIC.equals(authType)) { if (config.getAuthenticationConfig().getBasicConfig().getUsername() == null || config.getAuthenticationConfig().getBasicConfig().getPassword() == null) { - throw new RuntimeException("Jira ID or Credential are required for Basic AuthType"); + throw new RuntimeException("Confluence ID or Credential are required for Basic AuthType"); } } diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/Constants.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/Constants.java index dafa4c3f90..114c33cebd 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/Constants.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/Constants.java @@ -19,14 +19,12 @@ public class Constants { public static final String KEY = "key"; public static final String NAME = "name"; - public static final String PROJECT = "project"; + public static final String SPACE = "space"; public static final String OAUTH2 = "OAuth2"; - public static final String _PROJECT = "project-"; - public static final String _ISSUE = "ISSUE-"; - public static final String UPDATED = "updated"; - public static final String PROJECT_KEY = "j_project_key"; - public static final String PROJECT_NAME = "j_project_name"; - public static final String ISSUE_KEY = "j_issue_key"; + public static final String LAST_MODIFIED = "lastModified"; + public static final String SPACE_KEY = "space"; + public static final String CONTENT_TITLE = "title"; + public static final String CONTENT_ID = "id"; public static final String CREATED = "created"; public static final String BASIC = "Basic"; public static final String LIVE = "live"; diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/JqlConstants.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/CqlConstants.java similarity index 55% rename from data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/JqlConstants.java rename to data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/CqlConstants.java index 7dfe523df2..74b3a0f6c7 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/JqlConstants.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/CqlConstants.java @@ -10,21 +10,19 @@ package org.opensearch.dataprepper.plugins.source.confluence.utils; -public class JqlConstants { +public class CqlConstants { public static final String GREATER_THAN_EQUALS = ">="; public static final String CLOSING_ROUND_BRACKET = ")"; public static final String SLASH = "/"; - public static final String PROJECT_IN = " AND project in ("; - public static final String STATUS_IN = " AND status in ("; - public static final String PROJECT_NOT_IN = " AND project not in ("; - public static final String STATUS_NOT_IN = " AND status not in ("; + public static final String SPACE_IN = " AND space in ("; + public static final String SPACE_NOT_IN = " AND space not in ("; public static final String DELIMITER = "\",\""; public static final String PREFIX = "\""; public static final String SUFFIX = "\""; - public static final String ISSUE_TYPE_IN = " AND issueType in ("; - public static final String ISSUE_TYPE_NOT_IN = " AND issueType not in ("; - public static final String JQL_FIELD = "jql"; + public static final String CONTENT_TYPE_IN = " AND type in ("; + public static final String CONTENT_TYPE_NOT_IN = " AND type not in ("; + public static final String CQL_FIELD = "cql"; public static final String EXPAND_FIELD = "expand"; - public static final String EXPAND_VALUE = "all"; + public static final String EXPAND_VALUE = "all,space"; } diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceClientTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceClientTest.java index a75e6dfbc2..efd8e20303 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceClientTest.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceClientTest.java @@ -22,10 +22,6 @@ import org.opensearch.dataprepper.model.buffer.Buffer; import org.opensearch.dataprepper.model.event.Event; import org.opensearch.dataprepper.model.record.Record; -import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceClient; -import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceIterator; -import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceService; -import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceSourceConfig; import org.opensearch.dataprepper.plugins.source.source_crawler.base.PluginExecutorServiceProvider; import org.opensearch.dataprepper.plugins.source.source_crawler.coordination.state.SaasWorkerProgressState; @@ -92,7 +88,7 @@ void testExecutePartition() throws Exception { Instant exportStartTime = Instant.now(); when(saasWorkerProgressState.getExportStartTime()).thenReturn(Instant.ofEpochSecond(exportStartTime.toEpochMilli())); - when(confluenceService.getIssue(anyString())).thenReturn("{\"id\":\"ID1\",\"key\":\"TEST-1\"}"); + when(confluenceService.getContent(anyString())).thenReturn("{\"id\":\"ID1\",\"key\":\"TEST-1\"}"); ArgumentCaptor>> recordsCaptor = ArgumentCaptor.forClass((Class) Collection.class); @@ -117,7 +113,7 @@ void testExecutePartitionError() throws Exception { Instant exportStartTime = Instant.now(); when(saasWorkerProgressState.getExportStartTime()).thenReturn(Instant.ofEpochSecond(exportStartTime.toEpochMilli())); - when(confluenceService.getIssue(anyString())).thenReturn("{\"id\":\"ID1\",\"key\":\"TEST-1\"}"); + when(confluenceService.getContent(anyString())).thenReturn("{\"id\":\"ID1\",\"key\":\"TEST-1\"}"); ObjectMapper mockObjectMapper = mock(ObjectMapper.class); @@ -139,7 +135,7 @@ void bufferWriteRuntimeTest() throws Exception { Instant exportStartTime = Instant.now(); when(saasWorkerProgressState.getExportStartTime()).thenReturn(Instant.ofEpochSecond(exportStartTime.toEpochMilli())); - when(confluenceService.getIssue(anyString())).thenReturn("{\"id\":\"ID1\",\"key\":\"TEST-1\"}"); + when(confluenceService.getContent(anyString())).thenReturn("{\"id\":\"ID1\",\"key\":\"TEST-1\"}"); ArgumentCaptor>> recordsCaptor = ArgumentCaptor.forClass((Class) Collection.class); diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceConfigHelperTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceConfigHelperTest.java index d83967cf0b..57ea307b5c 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceConfigHelperTest.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceConfigHelperTest.java @@ -44,9 +44,6 @@ public class ConfluenceConfigHelperTest { @Mock FilterConfig filterConfig; - @Mock - StatusConfig statusConfig; - @Mock PageTypeConfig pageTypeConfig; @@ -76,20 +73,20 @@ void testInitialization() { ConfluenceConfigHelper confluenceConfigHelper = new ConfluenceConfigHelper(); assertNotNull(confluenceConfigHelper); } - + @Test void testGetIssueTypeFilter() { when(confluenceSourceConfig.getFilterConfig()).thenReturn(filterConfig); when(filterConfig.getPageTypeConfig()).thenReturn(pageTypeConfig); - assertTrue(ConfluenceConfigHelper.getIssueTypeIncludeFilter(confluenceSourceConfig).isEmpty()); - assertTrue(ConfluenceConfigHelper.getIssueTypeExcludeFilter(confluenceSourceConfig).isEmpty()); + assertTrue(ConfluenceConfigHelper.getContentTypeIncludeFilter(confluenceSourceConfig).isEmpty()); + assertTrue(ConfluenceConfigHelper.getContentTypeExcludeFilter(confluenceSourceConfig).isEmpty()); List issueTypeFilter = List.of("Bug", "Story"); List issueTypeExcludeFilter = List.of("Bug2", "Story2"); when(pageTypeConfig.getInclude()).thenReturn(issueTypeFilter); when(pageTypeConfig.getExclude()).thenReturn(issueTypeExcludeFilter); - assertEquals(issueTypeFilter, ConfluenceConfigHelper.getIssueTypeIncludeFilter(confluenceSourceConfig)); - assertEquals(issueTypeExcludeFilter, ConfluenceConfigHelper.getIssueTypeExcludeFilter(confluenceSourceConfig)); + assertEquals(issueTypeFilter, ConfluenceConfigHelper.getContentTypeIncludeFilter(confluenceSourceConfig)); + assertEquals(issueTypeExcludeFilter, ConfluenceConfigHelper.getContentTypeExcludeFilter(confluenceSourceConfig)); } @Test @@ -97,14 +94,14 @@ void testGetProjectNameFilter() { when(confluenceSourceConfig.getFilterConfig()).thenReturn(filterConfig); when(filterConfig.getSpaceConfig()).thenReturn(spaceConfig); when(spaceConfig.getNameConfig()).thenReturn(nameConfig); - assertTrue(ConfluenceConfigHelper.getProjectNameIncludeFilter(confluenceSourceConfig).isEmpty()); - assertTrue(ConfluenceConfigHelper.getProjectNameExcludeFilter(confluenceSourceConfig).isEmpty()); + assertTrue(ConfluenceConfigHelper.getSpacesNameIncludeFilter(confluenceSourceConfig).isEmpty()); + assertTrue(ConfluenceConfigHelper.getSpacesNameExcludeFilter(confluenceSourceConfig).isEmpty()); List projectNameFilter = List.of("TEST", "TEST2"); List projectNameExcludeFilter = List.of("TEST3", "TEST4"); when(nameConfig.getInclude()).thenReturn(projectNameFilter); when(nameConfig.getExclude()).thenReturn(projectNameExcludeFilter); - assertEquals(projectNameFilter, ConfluenceConfigHelper.getProjectNameIncludeFilter(confluenceSourceConfig)); - assertEquals(projectNameExcludeFilter, ConfluenceConfigHelper.getProjectNameExcludeFilter(confluenceSourceConfig)); + assertEquals(projectNameFilter, ConfluenceConfigHelper.getSpacesNameIncludeFilter(confluenceSourceConfig)); + assertEquals(projectNameExcludeFilter, ConfluenceConfigHelper.getSpacesNameExcludeFilter(confluenceSourceConfig)); } diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceItemInfoTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceItemInfoTest.java index f10249b686..26ef017dec 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceItemInfoTest.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceItemInfoTest.java @@ -15,7 +15,6 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceItemInfo; import org.opensearch.dataprepper.plugins.source.confluence.utils.Constants; import java.time.Instant; @@ -59,8 +58,8 @@ void setUP() { void testGetters() { assertEquals(confluenceItemInfo.getItemId(), itemId); assertEquals(confluenceItemInfo.getId(), id); - assertEquals(confluenceItemInfo.getProject(), project); - assertEquals(confluenceItemInfo.getIssueType(), issueType); + assertEquals(confluenceItemInfo.getSpace(), project); + assertEquals(confluenceItemInfo.getContentType(), issueType); assertEquals(confluenceItemInfo.getMetadata(), metadata); assertEquals(confluenceItemInfo.getEventTime(), eventTime); } @@ -78,12 +77,12 @@ void testSetter() { assertNotEquals(confluenceItemInfo.getItemId(), itemId); confluenceItemInfo.setId("newID"); assertNotEquals(confluenceItemInfo.getId(), id); - confluenceItemInfo.setProject("newProject"); - assertNotEquals(confluenceItemInfo.getProject(), project); + confluenceItemInfo.setSpace("newProject"); + assertNotEquals(confluenceItemInfo.getSpace(), project); confluenceItemInfo.setMetadata(newMetadata); assertNotEquals(confluenceItemInfo.getMetadata(), metadata); - confluenceItemInfo.setIssueType("newIssueType"); - assertNotEquals(confluenceItemInfo.getIssueType(), issueType); + confluenceItemInfo.setContentType("newIssueType"); + assertNotEquals(confluenceItemInfo.getContentType(), issueType); } @@ -97,11 +96,11 @@ void testGetPartitionKey() { @Test void testGetLastModifiedAt() { - when(metadata.get(Constants.UPDATED)).thenReturn("5"); + when(metadata.get(Constants.LAST_MODIFIED)).thenReturn("5"); when(metadata.get(Constants.CREATED)).thenReturn("0"); assertEquals(Instant.ofEpochMilli(5), confluenceItemInfo.getLastModifiedAt()); - when(metadata.get(Constants.UPDATED)).thenReturn("5"); + when(metadata.get(Constants.LAST_MODIFIED)).thenReturn("5"); when(metadata.get(Constants.CREATED)).thenReturn("7"); assertEquals(Instant.ofEpochMilli(7), confluenceItemInfo.getLastModifiedAt()); } diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceIteratorTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceIteratorTest.java index ac64972c72..cc2bfde044 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceIteratorTest.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceIteratorTest.java @@ -15,38 +15,26 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.dataprepper.plugins.source.confluence.models.IssueBean; -import org.opensearch.dataprepper.plugins.source.confluence.models.SearchResults; +import org.opensearch.dataprepper.plugins.source.confluence.models.ConfluenceSearchResults; import org.opensearch.dataprepper.plugins.source.confluence.rest.ConfluenceRestClient; import org.opensearch.dataprepper.plugins.source.source_crawler.base.PluginExecutorServiceProvider; import java.time.Instant; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.UUID; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.when; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.CREATED; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.KEY; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.UPDATED; @ExtendWith(MockitoExtension.class) public class ConfluenceIteratorTest { @Mock - private SearchResults mockSearchResults; + private ConfluenceSearchResults mockConfluenceSearchResults; @Mock private ConfluenceRestClient confluenceRestClient; private ConfluenceService confluenceService; @@ -69,9 +57,7 @@ void testInitialization() { confluenceIterator = createObjectUnderTest(); assertNotNull(confluenceIterator); confluenceIterator.initialize(Instant.ofEpochSecond(0)); - when(mockSearchResults.getIssues()).thenReturn(new ArrayList<>()); - when(mockSearchResults.getTotal()).thenReturn(0); - doReturn(mockSearchResults).when(confluenceRestClient).getAllIssues(any(StringBuilder.class), anyInt(), any(ConfluenceSourceConfig.class)); + doReturn(mockConfluenceSearchResults).when(confluenceRestClient).getAllContent(any(StringBuilder.class), anyInt()); assertFalse(confluenceIterator.hasNext()); } @@ -94,21 +80,6 @@ void sleepInterruptionTest() { testThread.interrupt(); } - @Test - void testItemInfoQueueNotEmpty() { - confluenceIterator = createObjectUnderTest(); - List mockIssues = new ArrayList<>(); - IssueBean issue1 = createIssueBean(false); - mockIssues.add(issue1); - when(mockSearchResults.getIssues()).thenReturn(mockIssues); - when(mockSearchResults.getTotal()).thenReturn(0); - doReturn(mockSearchResults).when(confluenceRestClient).getAllIssues(any(StringBuilder.class), anyInt(), any(ConfluenceSourceConfig.class)); - - confluenceIterator.initialize(Instant.ofEpochSecond(0)); - confluenceIterator.setCrawlerQWaitTimeMillis(1); - assertTrue(confluenceIterator.hasNext()); - assertNotNull(confluenceIterator.next()); - } @Test void testStartCrawlerThreads() { @@ -119,83 +90,5 @@ void testStartCrawlerThreads() { assertEquals(1, confluenceIterator.showFutureList().size()); } - @Test - void testFuturesCompleted() throws InterruptedException { - confluenceIterator = createObjectUnderTest(); - List mockIssues = new ArrayList<>(); - IssueBean issue1 = createIssueBean(false); - mockIssues.add(issue1); - IssueBean issue2 = createIssueBean(false); - mockIssues.add(issue2); - IssueBean issue3 = createIssueBean(false); - mockIssues.add(issue3); - when(mockSearchResults.getIssues()).thenReturn(mockIssues); - when(mockSearchResults.getTotal()).thenReturn(0); - doReturn(mockSearchResults).when(confluenceRestClient).getAllIssues(any(StringBuilder.class), anyInt(), any(ConfluenceSourceConfig.class)); - confluenceIterator.initialize(Instant.ofEpochSecond(0)); - confluenceIterator.setCrawlerQWaitTimeMillis(1); - confluenceIterator.hasNext(); - - Thread.sleep(1); - confluenceIterator.showFutureList().forEach(future -> assertTrue(future.isDone())); - assertEquals(confluenceIterator.showItemInfoQueue().size(), mockIssues.size()); - } - - @Test - void testItemInfoQueueEmpty() { - confluenceIterator = createObjectUnderTest(); - List mockIssues = new ArrayList<>(); - when(mockSearchResults.getIssues()).thenReturn(mockIssues); - when(mockSearchResults.getTotal()).thenReturn(0); - doReturn(mockSearchResults).when(confluenceRestClient).getAllIssues(any(StringBuilder.class), anyInt(), any(ConfluenceSourceConfig.class)); - - confluenceIterator.initialize(Instant.ofEpochSecond(0)); - confluenceIterator.setCrawlerQWaitTimeMillis(1); - assertFalse(confluenceIterator.hasNext()); - assertThrows(NoSuchElementException.class, () -> confluenceIterator.next()); - } - - - private IssueBean createIssueBean(boolean nullFields) { - IssueBean issue1 = new IssueBean(); - issue1.setId(UUID.randomUUID().toString()); - issue1.setKey("issue_1_key"); - issue1.setSelf("https://example.com/rest/api/2/issue/123"); - issue1.setExpand("operations,versionedRepresentations,editmeta"); - - Map fieldMap = new HashMap<>(); - if (!nullFields) { - fieldMap.put(CREATED, Instant.now()); - fieldMap.put(UPDATED, Instant.now()); - } else { - fieldMap.put(CREATED, 0); - fieldMap.put(UPDATED, 0); - } - - Map issueTypeMap = new HashMap<>(); - issueTypeMap.put("name", "Task"); - issueTypeMap.put("self", "https://example.com/rest/api/2/issuetype/1"); - issueTypeMap.put("id", "1"); - fieldMap.put("issuetype", issueTypeMap); - - Map projectMap = new HashMap<>(); - if (!nullFields) { - projectMap.put("name", "project name test"); - projectMap.put(KEY, "TEST"); - } - fieldMap.put("project", projectMap); - - Map priorityMap = new HashMap<>(); - priorityMap.put("name", "Medium"); - fieldMap.put("priority", priorityMap); - - Map statusMap = new HashMap<>(); - statusMap.put("name", "In Progress"); - fieldMap.put("status", statusMap); - - issue1.setFields(fieldMap); - - return issue1; - } } diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceServiceTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceServiceTest.java index 9859ea3051..df4b9d32be 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceServiceTest.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceServiceTest.java @@ -21,8 +21,8 @@ import org.opensearch.dataprepper.model.plugin.PluginConfigVariable; import org.opensearch.dataprepper.plugins.source.confluence.configuration.Oauth2Config; import org.opensearch.dataprepper.plugins.source.confluence.exception.BadRequestException; -import org.opensearch.dataprepper.plugins.source.confluence.models.IssueBean; -import org.opensearch.dataprepper.plugins.source.confluence.models.SearchResults; +import org.opensearch.dataprepper.plugins.source.confluence.models.ConfluenceItem; +import org.opensearch.dataprepper.plugins.source.confluence.models.ConfluenceSearchResults; import org.opensearch.dataprepper.plugins.source.confluence.rest.ConfluenceRestClient; import org.opensearch.dataprepper.plugins.source.confluence.utils.MockPluginConfigVariableImpl; import org.opensearch.dataprepper.plugins.source.source_crawler.base.PluginExecutorServiceProvider; @@ -58,10 +58,10 @@ import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.BASIC; import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.CREATED; import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.KEY; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.LAST_MODIFIED; import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.NAME; import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.OAUTH2; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.PROJECT; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.UPDATED; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.SPACE; /** @@ -178,12 +178,12 @@ void testJiraServiceInitialization() throws JsonProcessingException { ConfluenceSourceConfig confluenceSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey); ConfluenceService confluenceService = new ConfluenceService(confluenceSourceConfig, confluenceRestClient); assertNotNull(confluenceService); - when(confluenceRestClient.getIssue(anyString())).thenReturn("test String"); - assertNotNull(confluenceService.getIssue("test Key")); + when(confluenceRestClient.getContent(anyString())).thenReturn("test String"); + assertNotNull(confluenceService.getContent("test Key")); } @Test - public void testGetJiraEntities() throws JsonProcessingException { + public void testGetPages() throws JsonProcessingException { List issueType = new ArrayList<>(); List issueStatus = new ArrayList<>(); List projectKey = new ArrayList<>(); @@ -192,23 +192,21 @@ public void testGetJiraEntities() throws JsonProcessingException { projectKey.add("KAN"); ConfluenceSourceConfig confluenceSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey); ConfluenceService confluenceService = spy(new ConfluenceService(confluenceSourceConfig, confluenceRestClient)); - List mockIssues = new ArrayList<>(); - IssueBean issue1 = createIssueBean(false, false); + List mockIssues = new ArrayList<>(); + ConfluenceItem issue1 = createConfluenceItemBean(false, false); mockIssues.add(issue1); - IssueBean issue2 = createIssueBean(true, false); + ConfluenceItem issue2 = createConfluenceItemBean(true, false); mockIssues.add(issue2); - IssueBean issue3 = createIssueBean(false, true); + ConfluenceItem issue3 = createConfluenceItemBean(false, true); mockIssues.add(issue3); - SearchResults mockSearchResults = mock(SearchResults.class); - when(mockSearchResults.getIssues()).thenReturn(mockIssues); - when(mockSearchResults.getTotal()).thenReturn(mockIssues.size()); + ConfluenceSearchResults mockConfluenceSearchResults = mock(ConfluenceSearchResults.class); - doReturn(mockSearchResults).when(confluenceRestClient).getAllIssues(any(StringBuilder.class), anyInt(), any(ConfluenceSourceConfig.class)); + doReturn(mockConfluenceSearchResults).when(confluenceRestClient).getAllContent(any(StringBuilder.class), anyInt()); Instant timestamp = Instant.ofEpochSecond(0); Queue itemInfoQueue = new ConcurrentLinkedQueue<>(); - confluenceService.getJiraEntities(confluenceSourceConfig, timestamp, itemInfoQueue); + confluenceService.getPages(confluenceSourceConfig, timestamp, itemInfoQueue); assertEquals(mockIssues.size(), itemInfoQueue.size()); } @@ -220,21 +218,20 @@ public void buildIssueItemInfoMultipleFutureThreads() throws JsonProcessingExcep issueType.add("Task"); ConfluenceSourceConfig confluenceSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey); ConfluenceService confluenceService = spy(new ConfluenceService(confluenceSourceConfig, confluenceRestClient)); - List mockIssues = new ArrayList<>(); + List mockIssues = new ArrayList<>(); for (int i = 0; i < 50; i++) { - IssueBean issue1 = createIssueBean(false, false); + ConfluenceItem issue1 = createConfluenceItemBean(false, false); mockIssues.add(issue1); } - SearchResults mockSearchResults = mock(SearchResults.class); - when(mockSearchResults.getIssues()).thenReturn(mockIssues); - when(mockSearchResults.getTotal()).thenReturn(100); + ConfluenceSearchResults mockConfluenceSearchResults = mock(ConfluenceSearchResults.class); - doReturn(mockSearchResults).when(confluenceRestClient).getAllIssues(any(StringBuilder.class), anyInt(), any(ConfluenceSourceConfig.class)); + + doReturn(mockConfluenceSearchResults).when(confluenceRestClient).getAllContent(any(StringBuilder.class), anyInt()); Instant timestamp = Instant.ofEpochSecond(0); Queue itemInfoQueue = new ConcurrentLinkedQueue<>(); - confluenceService.getJiraEntities(confluenceSourceConfig, timestamp, itemInfoQueue); + confluenceService.getPages(confluenceSourceConfig, timestamp, itemInfoQueue); assertTrue(itemInfoQueue.size() >= 100); } @@ -256,11 +253,11 @@ public void testBadProjectKeys() throws JsonProcessingException { Instant timestamp = Instant.ofEpochSecond(0); Queue itemInfoQueue = new ConcurrentLinkedQueue<>(); - assertThrows(BadRequestException.class, () -> confluenceService.getJiraEntities(confluenceSourceConfig, timestamp, itemInfoQueue)); + assertThrows(BadRequestException.class, () -> confluenceService.getPages(confluenceSourceConfig, timestamp, itemInfoQueue)); } @Test - public void testGetJiraEntitiesException() throws JsonProcessingException { + public void testGetPagesException() throws JsonProcessingException { List issueType = new ArrayList<>(); List issueStatus = new ArrayList<>(); List projectKey = new ArrayList<>(); @@ -268,29 +265,27 @@ public void testGetJiraEntitiesException() throws JsonProcessingException { ConfluenceSourceConfig confluenceSourceConfig = createJiraConfiguration(BASIC, issueType, issueStatus, projectKey); ConfluenceService confluenceService = spy(new ConfluenceService(confluenceSourceConfig, confluenceRestClient)); - doThrow(RuntimeException.class).when(confluenceRestClient).getAllIssues(any(StringBuilder.class), anyInt(), any(ConfluenceSourceConfig.class)); + doThrow(RuntimeException.class).when(confluenceRestClient).getAllContent(any(StringBuilder.class), anyInt()); Instant timestamp = Instant.ofEpochSecond(0); Queue itemInfoQueue = new ConcurrentLinkedQueue<>(); - assertThrows(RuntimeException.class, () -> confluenceService.getJiraEntities(confluenceSourceConfig, timestamp, itemInfoQueue)); + assertThrows(RuntimeException.class, () -> confluenceService.getPages(confluenceSourceConfig, timestamp, itemInfoQueue)); } - private IssueBean createIssueBean(boolean nullFields, boolean createdNull) { - IssueBean issue1 = new IssueBean(); + private ConfluenceItem createConfluenceItemBean(boolean nullFields, boolean createdNull) { + ConfluenceItem issue1 = new ConfluenceItem(); issue1.setId(UUID.randomUUID().toString()); - issue1.setKey("issue_1_key"); - issue1.setSelf("https://example.com/rest/api/2/issue/123"); - issue1.setExpand("operations,versionedRepresentations,editmeta"); + issue1.setTitle("issue_1_key"); Map fieldMap = new HashMap<>(); if (!nullFields) { fieldMap.put(CREATED, "2024-07-06T21:12:23.437-0700"); - fieldMap.put(UPDATED, "2024-07-06T21:12:23.106-0700"); + fieldMap.put(LAST_MODIFIED, "2024-07-06T21:12:23.106-0700"); } else { fieldMap.put(CREATED, 0); - fieldMap.put(UPDATED, 0); + fieldMap.put(LAST_MODIFIED, 0); } if (createdNull) { fieldMap.put(CREATED, null); @@ -307,7 +302,7 @@ private IssueBean createIssueBean(boolean nullFields, boolean createdNull) { projectMap.put(NAME, "project name test"); projectMap.put(KEY, "TEST"); } - fieldMap.put(PROJECT, projectMap); + fieldMap.put(SPACE, projectMap); Map priorityMap = new HashMap<>(); priorityMap.put(NAME, "Medium"); @@ -317,8 +312,6 @@ private IssueBean createIssueBean(boolean nullFields, boolean createdNull) { statusMap.put(NAME, "In Progress"); fieldMap.put("statuses", statusMap); - issue1.setFields(fieldMap); - return issue1; } diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceConfigTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceConfigTest.java index ddf84b8004..73fdbe9dbe 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceConfigTest.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceConfigTest.java @@ -114,7 +114,6 @@ void testGetters() throws Exception { confluenceSourceConfig = createJiraSourceConfig(BASIC, false); assertEquals(confluenceSourceConfig.getFilterConfig().getPageTypeConfig().getInclude(), issueTypeList); assertEquals(confluenceSourceConfig.getFilterConfig().getSpaceConfig().getNameConfig().getInclude(), projectList); - assertEquals(confluenceSourceConfig.getFilterConfig().getStatusConfig().getInclude(), statusList); assertEquals(confluenceSourceConfig.getAccountUrl(), accountUrl); assertEquals(confluenceSourceConfig.getAuthenticationConfig().getBasicConfig().getPassword(), password); assertEquals(confluenceSourceConfig.getAuthenticationConfig().getBasicConfig().getUsername(), username); diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceItemTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceItemTest.java new file mode 100644 index 0000000000..1c22fb673a --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceItemTest.java @@ -0,0 +1,71 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.models; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +@ExtendWith(MockitoExtension.class) +public class ConfluenceItemTest { + + + private ConfluenceItem issueBean; + + @BeforeEach + void setup() { + issueBean = new ConfluenceItem(); + } + + @Test + public void testInitialization() { + assertNotNull(issueBean); + } + + @Test + public void testNull() { + assertNull(issueBean.getId()); + } + + @Test + void testNullCases() { + assertEquals(issueBean.getUpdatedTimeMillis(), 0); + } + + @Test + void testGivenDateField() { + Map fieldsTestObject = new HashMap<>(); + fieldsTestObject.put("created", "2024-07-06T21:12:23.437-0700"); + fieldsTestObject.put("updated", "2022-07-06T21:12:23.106-0700"); + assertEquals(issueBean.getCreatedTimeMillis(), 1720325543000L); + assertEquals(issueBean.getUpdatedTimeMillis(), 1657167143000L); + } + + @Test + public void testStringSettersAndGetters() { + String self = "selfTest"; + String key = "keyTest"; + String id = "idTest"; + String expand = "expandTest"; + + issueBean.setId(id); + assertEquals(issueBean.getId(), id); + } + +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceSearchResultsTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceSearchResultsTest.java new file mode 100644 index 0000000000..fe070c8b98 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceSearchResultsTest.java @@ -0,0 +1,41 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.models; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; + +import static org.junit.jupiter.api.Assertions.assertNotNull; + +@ExtendWith(MockitoExtension.class) +public class ConfluenceSearchResultsTest { + + private ConfluenceSearchResults confluenceSearchResults; + private final ObjectMapper objectMapper = new ObjectMapper(); + + @BeforeEach + public void setUp() throws JsonProcessingException { + String state = "{}"; + confluenceSearchResults = objectMapper.readValue(state, ConfluenceSearchResults.class); + } + + @Test + public void testConstructor() { + assertNotNull(confluenceSearchResults); + + } + + +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/IssueBeanTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/IssueBeanTest.java deleted file mode 100644 index 3b3216a03a..0000000000 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/IssueBeanTest.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - */ - -package org.opensearch.dataprepper.plugins.source.confluence.models; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.dataprepper.plugins.source.confluence.models.IssueBean; - -import java.util.HashMap; -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.KEY; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.NAME; -import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.PROJECT; - -@ExtendWith(MockitoExtension.class) -public class IssueBeanTest { - - @Mock - private Map renderedFieldsTestObject; - - @Mock - private Map propertiesTestObject; - - @Mock - private Map namesTestObject; - - @Mock - private Map fieldsTestObject; - - private IssueBean issueBean; - - @BeforeEach - void setup() { - issueBean = new IssueBean(); - } - - @Test - public void testInitialization() { - assertNotNull(issueBean); - } - - @Test - public void testNull() { - assertNull(issueBean.getExpand()); - assertNull(issueBean.getId()); - assertNull(issueBean.getSelf()); - assertNull(issueBean.getKey()); - assertNull(issueBean.getRenderedFields()); - assertNull(issueBean.getProperties()); - assertNull(issueBean.getNames()); - assertNull(issueBean.getFields()); - } - - @Test - void testNullCases() { - assertNull(issueBean.getProject()); - assertNull(issueBean.getProjectName()); - assertEquals(issueBean.getUpdatedTimeMillis(), 0); - } - - @Test - void testGivenDateField() { - Map fieldsTestObject = new HashMap<>(); - fieldsTestObject.put("created", "2024-07-06T21:12:23.437-0700"); - fieldsTestObject.put("updated", "2022-07-06T21:12:23.106-0700"); - issueBean.setFields(fieldsTestObject); - assertEquals(issueBean.getCreatedTimeMillis(), 1720325543000L); - assertEquals(issueBean.getUpdatedTimeMillis(), 1657167143000L); - } - - @Test - public void testStringSettersAndGetters() { - String self = "selfTest"; - String key = "keyTest"; - String id = "idTest"; - String expand = "expandTest"; - - issueBean.setExpand(expand); - assertEquals(issueBean.getExpand(), expand); - issueBean.setId(id); - assertEquals(issueBean.getId(), id); - issueBean.setSelf(self); - assertEquals(issueBean.getSelf(), self); - issueBean.setKey(key); - assertEquals(issueBean.getKey(), key); - } - - @Test - public void testMapSettersAndGetters() { - - issueBean.setRenderedFields(renderedFieldsTestObject); - assertEquals(issueBean.getRenderedFields(), renderedFieldsTestObject); - issueBean.setProperties(propertiesTestObject); - assertEquals(issueBean.getProperties(), propertiesTestObject); - issueBean.setNames(namesTestObject); - assertEquals(issueBean.getNames(), namesTestObject); - issueBean.setFields(fieldsTestObject); - assertEquals(issueBean.getFields(), fieldsTestObject); - } - - @Test - public void testFieldPropertyGetters() { - Map fieldsTestObject = new HashMap<>(); - Map projectTestObject = new HashMap<>(); - String projectName = "name of project"; - String projectKey = "PROJKEY"; - projectTestObject.put(KEY, projectKey); - projectTestObject.put(NAME, projectName); - fieldsTestObject.put(PROJECT, projectTestObject); - - issueBean.setFields(fieldsTestObject); - assertEquals(projectKey, issueBean.getProject()); - assertEquals(projectName, issueBean.getProjectName()); - } - -} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/SearchResultsTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/SearchResultsTest.java deleted file mode 100644 index 44321d2634..0000000000 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/SearchResultsTest.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - */ - -package org.opensearch.dataprepper.plugins.source.confluence.models; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.dataprepper.plugins.source.confluence.models.IssueBean; -import org.opensearch.dataprepper.plugins.source.confluence.models.SearchResults; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; - -@ExtendWith(MockitoExtension.class) -public class SearchResultsTest { - - private SearchResults searchResults; - private final ObjectMapper objectMapper = new ObjectMapper(); - - @BeforeEach - public void setUp() throws JsonProcessingException { - String state = "{}"; - searchResults = objectMapper.readValue(state, SearchResults.class); - } - - @Test - public void testConstructor() { - assertNotNull(searchResults); - - assertNull(searchResults.getExpand()); - assertNull(searchResults.getStartAt()); - assertNull(searchResults.getMaxResults()); - assertNull(searchResults.getTotal()); - assertNull(searchResults.getIssues()); - } - - @Test - public void testGetters() throws JsonProcessingException { - String expand = "expandTest"; - Integer startAt = 1; - Integer maxResults = 100; - Integer total = 10; - List testIssues = new ArrayList<>(); - IssueBean issue1 = new IssueBean(); - IssueBean issue2 = new IssueBean(); - issue1.setId("issue 1"); - issue2.setId("issue 2"); - testIssues.add(issue1); - testIssues.add(issue2); - - - Map searchResultsMap = new HashMap<>(); - searchResultsMap.put("expand", expand); - searchResultsMap.put("startAt", startAt); - searchResultsMap.put("maxResults", maxResults); - searchResultsMap.put("total", total); - searchResultsMap.put("issues", testIssues); - - String jsonString = objectMapper.writeValueAsString(searchResultsMap); - - searchResults = objectMapper.readValue(jsonString, SearchResults.class); - - assertEquals(searchResults.getExpand(), expand); - assertEquals(searchResults.getStartAt(), startAt); - assertEquals(searchResults.getMaxResults(), maxResults); - assertEquals(searchResults.getTotal(), total); - - List returnedIssues = searchResults.getIssues(); - assertNotNull(returnedIssues); - assertEquals(testIssues.size(), returnedIssues.size()); - - for (int i = 0; i < testIssues.size(); i++) { - IssueBean originalIssue = testIssues.get(i); - IssueBean returnedIssue = returnedIssues.get(i); - - assertEquals(originalIssue.getId(), returnedIssue.getId()); - } - } - - -} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClientTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClientTest.java index 58863e85e5..27a0885208 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClientTest.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClientTest.java @@ -19,11 +19,12 @@ import org.junit.jupiter.params.provider.ValueSource; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceServiceTest; import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceSourceConfig; import org.opensearch.dataprepper.plugins.source.confluence.exception.BadRequestException; import org.opensearch.dataprepper.plugins.source.confluence.exception.UnAuthorizedException; -import org.opensearch.dataprepper.plugins.source.confluence.models.SearchResults; +import org.opensearch.dataprepper.plugins.source.confluence.models.ConfluenceSearchResults; import org.opensearch.dataprepper.plugins.source.confluence.rest.auth.ConfluenceAuthConfig; import org.opensearch.dataprepper.plugins.source.confluence.rest.auth.ConfluenceAuthFactory; import org.springframework.http.HttpStatus; @@ -58,6 +59,8 @@ public class ConfluenceRestClientTest { @Mock private ConfluenceAuthConfig authConfig; + private final PluginMetrics pluginMetrics = PluginMetrics.fromNames("jiraRestClient", "aws"); + private static Stream provideHttpStatusCodesWithExceptionClass() { return Stream.of( Arguments.of(HttpStatus.FORBIDDEN, UnAuthorizedException.class), @@ -74,24 +77,24 @@ public void testFetchingJiraIssue(String configFileName) { doReturn(new ResponseEntity<>(exampleTicketResponse, HttpStatus.OK)).when(restTemplate).getForEntity(any(URI.class), any(Class.class)); ConfluenceSourceConfig confluenceSourceConfig = ConfluenceServiceTest.createJiraConfigurationFromYaml(configFileName); ConfluenceAuthConfig authConfig = new ConfluenceAuthFactory(confluenceSourceConfig).getObject(); - ConfluenceRestClient confluenceRestClient = new ConfluenceRestClient(restTemplate, authConfig); - String ticketDetails = confluenceRestClient.getIssue("key"); + ConfluenceRestClient confluenceRestClient = new ConfluenceRestClient(restTemplate, authConfig, pluginMetrics); + String ticketDetails = confluenceRestClient.getContent("key"); assertEquals(exampleTicketResponse, ticketDetails); } @ParameterizedTest @MethodSource("provideHttpStatusCodesWithExceptionClass") void testInvokeRestApiTokenExpired(HttpStatus statusCode, Class expectedExceptionType) { - ConfluenceRestClient confluenceRestClient = new ConfluenceRestClient(restTemplate, authConfig); + ConfluenceRestClient confluenceRestClient = new ConfluenceRestClient(restTemplate, authConfig, pluginMetrics); confluenceRestClient.setSleepTimeMultiplier(1); when(authConfig.getUrl()).thenReturn("https://example.com/rest/api/2/issue/key"); when(restTemplate.getForEntity(any(URI.class), any(Class.class))).thenThrow(new HttpClientErrorException(statusCode)); - assertThrows(expectedExceptionType, () -> confluenceRestClient.getIssue("key")); + assertThrows(expectedExceptionType, () -> confluenceRestClient.getContent("key")); } @Test void testInvokeRestApiTokenExpiredInterruptException() throws InterruptedException { - ConfluenceRestClient confluenceRestClient = new ConfluenceRestClient(restTemplate, authConfig); + ConfluenceRestClient confluenceRestClient = new ConfluenceRestClient(restTemplate, authConfig, pluginMetrics); when(authConfig.getUrl()).thenReturn("https://example.com/rest/api/2/issue/key"); when(restTemplate.getForEntity(any(URI.class), any(Class.class))).thenThrow(new HttpClientErrorException(HttpStatus.TOO_MANY_REQUESTS)); confluenceRestClient.setSleepTimeMultiplier(100000); @@ -99,7 +102,7 @@ void testInvokeRestApiTokenExpiredInterruptException() throws InterruptedExcepti Thread testThread = new Thread(() -> { assertThrows(InterruptedException.class, () -> { try { - confluenceRestClient.getIssue("key"); + confluenceRestClient.getContent("key"); } catch (Exception e) { throw new RuntimeException(e); } @@ -111,40 +114,40 @@ void testInvokeRestApiTokenExpiredInterruptException() throws InterruptedExcepti } @Test - public void testGetAllIssuesOauth2() throws JsonProcessingException { + public void testGetAllContentOauth2() throws JsonProcessingException { List issueType = new ArrayList<>(); List issueStatus = new ArrayList<>(); List projectKey = new ArrayList<>(); issueType.add("Task"); ConfluenceSourceConfig confluenceSourceConfig = ConfluenceServiceTest.createJiraConfiguration(OAUTH2, issueType, issueStatus, projectKey); - ConfluenceRestClient confluenceRestClient = new ConfluenceRestClient(restTemplate, authConfig); - SearchResults mockSearchResults = mock(SearchResults.class); + ConfluenceRestClient confluenceRestClient = new ConfluenceRestClient(restTemplate, authConfig, pluginMetrics); + ConfluenceSearchResults mockConfluenceSearchResults = mock(ConfluenceSearchResults.class); doReturn("http://mock-service.jira.com/").when(authConfig).getUrl(); - doReturn(new ResponseEntity<>(mockSearchResults, HttpStatus.OK)).when(restTemplate).getForEntity(any(URI.class), any(Class.class)); - SearchResults results = confluenceRestClient.getAllIssues(jql, 0, confluenceSourceConfig); + doReturn(new ResponseEntity<>(mockConfluenceSearchResults, HttpStatus.OK)).when(restTemplate).getForEntity(any(URI.class), any(Class.class)); + ConfluenceSearchResults results = confluenceRestClient.getAllContent(jql, 0); assertNotNull(results); } @Test - public void testGetAllIssuesBasic() throws JsonProcessingException { + public void testGetAllContentBasic() throws JsonProcessingException { List issueType = new ArrayList<>(); List issueStatus = new ArrayList<>(); List projectKey = new ArrayList<>(); issueType.add("Task"); ConfluenceSourceConfig confluenceSourceConfig = ConfluenceServiceTest.createJiraConfiguration(BASIC, issueType, issueStatus, projectKey); - ConfluenceRestClient confluenceRestClient = new ConfluenceRestClient(restTemplate, authConfig); - SearchResults mockSearchResults = mock(SearchResults.class); + ConfluenceRestClient confluenceRestClient = new ConfluenceRestClient(restTemplate, authConfig, pluginMetrics); + ConfluenceSearchResults mockConfluenceSearchResults = mock(ConfluenceSearchResults.class); when(authConfig.getUrl()).thenReturn("https://example.com/"); - doReturn(new ResponseEntity<>(mockSearchResults, HttpStatus.OK)).when(restTemplate).getForEntity(any(URI.class), any(Class.class)); - SearchResults results = confluenceRestClient.getAllIssues(jql, 0, confluenceSourceConfig); + doReturn(new ResponseEntity<>(mockConfluenceSearchResults, HttpStatus.OK)).when(restTemplate).getForEntity(any(URI.class), any(Class.class)); + ConfluenceSearchResults results = confluenceRestClient.getAllContent(jql, 0); assertNotNull(results); } @Test public void testRestApiAddressValidation() throws JsonProcessingException { when(authConfig.getUrl()).thenReturn("https://224.0.0.1/"); - ConfluenceRestClient confluenceRestClient = new ConfluenceRestClient(restTemplate, authConfig); - assertThrows(BadRequestException.class, () -> confluenceRestClient.getIssue("TEST-1")); + ConfluenceRestClient confluenceRestClient = new ConfluenceRestClient(restTemplate, authConfig, pluginMetrics); + assertThrows(BadRequestException.class, () -> confluenceRestClient.getContent("TEST-1")); } } diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/utils/AddressValidationTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/utils/AddressValidationTest.java index 0077129427..862d01878e 100644 --- a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/utils/AddressValidationTest.java +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/utils/AddressValidationTest.java @@ -12,9 +12,6 @@ import org.junit.jupiter.api.Test; import org.opensearch.dataprepper.plugins.source.confluence.exception.BadRequestException; -import org.opensearch.dataprepper.plugins.source.confluence.utils.AddressValidation; -import org.opensearch.dataprepper.plugins.source.confluence.utils.Constants; -import org.opensearch.dataprepper.plugins.source.confluence.utils.JqlConstants; import java.net.InetAddress; import java.net.MalformedURLException; @@ -31,7 +28,7 @@ public class AddressValidationTest { void testInstanceCreation() { assertNotNull(new AddressValidation()); assertNotNull(new Constants()); - assertNotNull(new JqlConstants()); + assertNotNull(new CqlConstants()); } @Test