diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/build.gradle b/data-prepper-plugins/saas-source-plugins/atlassian-commons/build.gradle new file mode 100644 index 0000000000..21544050cb --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/build.gradle @@ -0,0 +1,26 @@ +plugins { + id 'java' +} + + +dependencies { + + implementation project(path: ':data-prepper-api') + implementation project(path: ':data-prepper-plugins:saas-source-plugins:source-crawler') + implementation project(path: ':data-prepper-plugins:common') + + implementation 'com.fasterxml.jackson.core:jackson-core' + implementation 'com.fasterxml.jackson.core:jackson-databind' + + implementation 'io.micrometer:micrometer-core' + implementation 'org.projectlombok:lombok:1.18.30' + annotationProcessor 'org.projectlombok:lombok:1.18.30' + implementation(libs.spring.web) + + implementation(libs.spring.context) { + exclude group: 'commons-logging', module: 'commons-logging' + } + + testImplementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.4' + testImplementation project(path: ':data-prepper-test-common') +} diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/AtlassianSourceConfig.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/AtlassianSourceConfig.java new file mode 100644 index 0000000000..bc661f96af --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/AtlassianSourceConfig.java @@ -0,0 +1,59 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.atlassian; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.Valid; +import lombok.Getter; +import org.opensearch.dataprepper.plugins.source.atlassian.configuration.AuthenticationConfig; +import org.opensearch.dataprepper.plugins.source.source_crawler.base.CrawlerSourceConfig; + +import java.util.List; + +@Getter +public class AtlassianSourceConfig implements CrawlerSourceConfig { + + private static final int DEFAULT_BATCH_SIZE = 50; + + /** + * Jira account url + */ + @JsonProperty("hosts") + protected List hosts; + + /** + * Authentication Config to Access Jira + */ + @JsonProperty("authentication") + @Valid + protected AuthenticationConfig authenticationConfig; + + /** + * Batch size for fetching tickets + */ + @JsonProperty("batch_size") + protected int batchSize = DEFAULT_BATCH_SIZE; + + + /** + * Boolean property indicating end to end acknowledgments state + */ + @JsonProperty("acknowledgments") + private boolean acknowledgments = false; + + public String getAccountUrl() { + return this.getHosts().get(0); + } + + public String getAuthType() { + return this.getAuthenticationConfig().getAuthType(); + } +} diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/configuration/AuthenticationConfig.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/configuration/AuthenticationConfig.java new file mode 100644 index 0000000000..0d1cf8b961 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/configuration/AuthenticationConfig.java @@ -0,0 +1,46 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.atlassian.configuration; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.Valid; +import jakarta.validation.constraints.AssertTrue; +import lombok.Getter; + +import static org.opensearch.dataprepper.plugins.source.atlassian.utils.Constants.BASIC; +import static org.opensearch.dataprepper.plugins.source.atlassian.utils.Constants.OAUTH2; + + +@Getter +public class AuthenticationConfig { + @JsonProperty("basic") + @Valid + private BasicConfig basicConfig; + + @JsonProperty("oauth2") + @Valid + private Oauth2Config oauth2Config; + + @AssertTrue(message = "Authentication config should have either basic or oauth2") + private boolean isValidAuthenticationConfig() { + boolean hasBasic = basicConfig != null; + boolean hasOauth = oauth2Config != null; + return hasBasic ^ hasOauth; + } + + public String getAuthType() { + if (basicConfig != null) { + return BASIC; + } else { + return OAUTH2; + } + } +} diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/configuration/BasicConfig.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/configuration/BasicConfig.java new file mode 100644 index 0000000000..a610be6e2a --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/configuration/BasicConfig.java @@ -0,0 +1,29 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.atlassian.configuration; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.constraints.AssertTrue; +import lombok.Getter; + +@Getter +public class BasicConfig { + @JsonProperty("username") + private String username; + + @JsonProperty("password") + private String password; + + @AssertTrue(message = "Username and Password are both required for Basic Auth") + private boolean isBasicConfigValid() { + return username != null && password != null; + } +} diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/configuration/Oauth2Config.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/configuration/Oauth2Config.java new file mode 100644 index 0000000000..812ef61afe --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/configuration/Oauth2Config.java @@ -0,0 +1,36 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.atlassian.configuration; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.constraints.AssertTrue; +import lombok.Getter; +import org.opensearch.dataprepper.model.plugin.PluginConfigVariable; + +@Getter +public class Oauth2Config { + @JsonProperty("client_id") + private String clientId; + + @JsonProperty("client_secret") + private String clientSecret; + + @JsonProperty("access_token") + private PluginConfigVariable accessToken; + + @JsonProperty("refresh_token") + private PluginConfigVariable refreshToken; + + @AssertTrue(message = "Client ID, Client Secret, Access Token, and Refresh Token are both required for Oauth2") + private boolean isOauth2ConfigValid() { + return clientId != null && clientSecret != null && accessToken != null && refreshToken != null; + } +} diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/AtlassianRestClient.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/AtlassianRestClient.java new file mode 100644 index 0000000000..5114091939 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/AtlassianRestClient.java @@ -0,0 +1,79 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.atlassian.rest; + +import com.google.common.annotations.VisibleForTesting; +import lombok.extern.slf4j.Slf4j; +import org.opensearch.dataprepper.plugins.source.atlassian.rest.auth.AtlassianAuthConfig; +import org.opensearch.dataprepper.plugins.source.source_crawler.exception.BadRequestException; +import org.opensearch.dataprepper.plugins.source.source_crawler.exception.UnauthorizedException; +import org.opensearch.dataprepper.plugins.source.source_crawler.utils.AddressValidation; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.RestTemplate; + +import java.net.URI; +import java.util.List; + +import static org.opensearch.dataprepper.logging.DataPrepperMarkers.NOISY; +import static org.opensearch.dataprepper.plugins.source.atlassian.utils.Constants.MAX_RETRIES; + +@Slf4j +public class AtlassianRestClient { + + public static final List RETRY_ATTEMPT_SLEEP_TIME = List.of(1, 2, 5, 10, 20, 40); + private int sleepTimeMultiplier = 1000; + private final RestTemplate restTemplate; + private final AtlassianAuthConfig authConfig; + + public AtlassianRestClient(RestTemplate restTemplate, AtlassianAuthConfig authConfig) { + this.restTemplate = restTemplate; + this.authConfig = authConfig; + } + + + protected ResponseEntity invokeRestApi(URI uri, Class responseType) throws BadRequestException { + AddressValidation.validateInetAddress(AddressValidation.getInetAddress(uri.toString())); + int retryCount = 0; + while (retryCount < MAX_RETRIES) { + try { + return restTemplate.getForEntity(uri, responseType); + } catch (HttpClientErrorException ex) { + HttpStatus statusCode = ex.getStatusCode(); + String statusMessage = ex.getMessage(); + log.error("An exception has occurred while getting response from Jira search API {}", ex.getMessage()); + if (statusCode == HttpStatus.FORBIDDEN) { + throw new UnauthorizedException(statusMessage); + } else if (statusCode == HttpStatus.UNAUTHORIZED) { + log.error(NOISY, "Token expired. We will try to renew the tokens now", ex); + authConfig.renewCredentials(); + } else if (statusCode == HttpStatus.TOO_MANY_REQUESTS) { + log.error(NOISY, "Hitting API rate limit. Backing off with sleep timer.", ex); + } + try { + Thread.sleep((long) RETRY_ATTEMPT_SLEEP_TIME.get(retryCount) * sleepTimeMultiplier); + } catch (InterruptedException e) { + throw new RuntimeException("Sleep in the retry attempt got interrupted", e); + } + } + retryCount++; + } + String errorMessage = String.format("Exceeded max retry attempts. Failed to execute the Rest API call %s", uri); + log.error(errorMessage); + throw new RuntimeException(errorMessage); + } + + @VisibleForTesting + public void setSleepTimeMultiplier(int multiplier) { + sleepTimeMultiplier = multiplier; + } +} diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/BasicAuthInterceptor.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/BasicAuthInterceptor.java new file mode 100644 index 0000000000..80c5e9729b --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/BasicAuthInterceptor.java @@ -0,0 +1,42 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.atlassian.rest; + +import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpRequest; +import org.springframework.http.client.ClientHttpRequestExecution; +import org.springframework.http.client.ClientHttpRequestInterceptor; +import org.springframework.http.client.ClientHttpResponse; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Base64; + + +public class BasicAuthInterceptor implements ClientHttpRequestInterceptor { + private final String username; + private final String password; + + public BasicAuthInterceptor(AtlassianSourceConfig config) { + this.username = config.getAuthenticationConfig().getBasicConfig().getUsername(); + this.password = config.getAuthenticationConfig().getBasicConfig().getPassword(); + } + + @Override + public ClientHttpResponse intercept(HttpRequest request, byte[] body, ClientHttpRequestExecution execution) throws IOException { + String auth = username + ":" + password; + byte[] encodedAuth = Base64.getEncoder().encode(auth.getBytes(StandardCharsets.US_ASCII)); + String authHeader = "Basic " + new String(encodedAuth); + request.getHeaders().set(HttpHeaders.AUTHORIZATION, authHeader); + return execution.execute(request, body); + } +} diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/CustomRestTemplateConfig.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/CustomRestTemplateConfig.java new file mode 100644 index 0000000000..a8cd1c131d --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/CustomRestTemplateConfig.java @@ -0,0 +1,42 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.atlassian.rest; + + +import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.rest.auth.AtlassianAuthConfig; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.http.client.ClientHttpRequestInterceptor; +import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; +import org.springframework.web.client.RestTemplate; + +import static org.opensearch.dataprepper.plugins.source.atlassian.utils.Constants.OAUTH2; + +@Configuration +public class CustomRestTemplateConfig { + + @Bean + public RestTemplate basicAuthRestTemplate(AtlassianSourceConfig config, AtlassianAuthConfig authConfig) { + RestTemplate restTemplate = new RestTemplate(); + restTemplate.setRequestFactory(new HttpComponentsClientHttpRequestFactory()); + ClientHttpRequestInterceptor httpInterceptor; + if (OAUTH2.equals(config.getAuthType())) { + httpInterceptor = new OAuth2RequestInterceptor(authConfig); + } else { + httpInterceptor = new BasicAuthInterceptor(config); + } + restTemplate.getInterceptors().add(httpInterceptor); + return restTemplate; + } + + +} diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/OAuth2RequestInterceptor.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/OAuth2RequestInterceptor.java new file mode 100644 index 0000000000..2eabfc87bd --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/OAuth2RequestInterceptor.java @@ -0,0 +1,36 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.atlassian.rest; + +import org.opensearch.dataprepper.plugins.source.atlassian.rest.auth.AtlassianAuthConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.rest.auth.AtlassianOauthConfig; +import org.springframework.http.HttpRequest; +import org.springframework.http.client.ClientHttpRequestExecution; +import org.springframework.http.client.ClientHttpRequestInterceptor; +import org.springframework.http.client.ClientHttpResponse; + +import java.io.IOException; + +public class OAuth2RequestInterceptor implements ClientHttpRequestInterceptor { + + private final AtlassianAuthConfig config; + + public OAuth2RequestInterceptor(AtlassianAuthConfig config) { + this.config = config; + } + + @Override + public ClientHttpResponse intercept(HttpRequest request, byte[] body, ClientHttpRequestExecution execution) throws IOException { + request.getHeaders().setBearerAuth(((AtlassianOauthConfig) config).getAccessToken()); + return execution.execute(request, body); + } + +} diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianAuthConfig.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianAuthConfig.java new file mode 100644 index 0000000000..71da3d6dd2 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianAuthConfig.java @@ -0,0 +1,35 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.atlassian.rest.auth; + +/** + * The interface that defines the behaviour for Jira auth configs. + */ +public interface AtlassianAuthConfig { + + /** + * Returns the URL for the Jira instance. + * + * @return the URL for the Jira instance. + */ + String getUrl(); + + /** + * Initializes the credentials for the Jira instance. + */ + default void initCredentials() { + } + + /** + * Renews the credentials for the Jira instance. + */ + void renewCredentials(); +} diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianAuthFactory.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianAuthFactory.java new file mode 100644 index 0000000000..e10f2e3d2a --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianAuthFactory.java @@ -0,0 +1,41 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.atlassian.rest.auth; + +import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig; +import org.springframework.beans.factory.FactoryBean; +import org.springframework.context.annotation.Configuration; + +import static org.opensearch.dataprepper.plugins.source.atlassian.utils.Constants.OAUTH2; + +@Configuration +public class AtlassianAuthFactory implements FactoryBean { + + private final AtlassianSourceConfig sourceConfig; + + public AtlassianAuthFactory(AtlassianSourceConfig sourceConfig) { + this.sourceConfig = sourceConfig; + } + + @Override + public AtlassianAuthConfig getObject() { + String authType = sourceConfig.getAuthType(); + if (OAUTH2.equals(authType)) { + return new AtlassianOauthConfig(sourceConfig); + } + return new AtlassianBasicAuthConfig(sourceConfig); + } + + @Override + public Class getObjectType() { + return AtlassianAuthConfig.class; + } +} diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianBasicAuthConfig.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianBasicAuthConfig.java new file mode 100644 index 0000000000..7b160ee65e --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianBasicAuthConfig.java @@ -0,0 +1,41 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.atlassian.rest.auth; + + +import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig; + +public class AtlassianBasicAuthConfig implements AtlassianAuthConfig { + + private String accountUrl; + private final AtlassianSourceConfig confluenceSourceConfig; + + public AtlassianBasicAuthConfig(AtlassianSourceConfig confluenceSourceConfig) { + this.confluenceSourceConfig = confluenceSourceConfig; + accountUrl = confluenceSourceConfig.getAccountUrl(); + if (!accountUrl.endsWith("/")) { + accountUrl += "/"; + } + } + + @Override + public String getUrl() { + return accountUrl; + } + + + @Override + public void renewCredentials() { + //do nothing for basic authentication + } + + +} diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianOauthConfig.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianOauthConfig.java new file mode 100644 index 0000000000..3608376259 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianOauthConfig.java @@ -0,0 +1,183 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.atlassian.rest.auth; + +import lombok.Getter; +import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.configuration.Oauth2Config; +import org.opensearch.dataprepper.plugins.source.source_crawler.exception.UnauthorizedException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.util.StringUtils; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.RestTemplate; + +import java.time.Instant; +import java.util.List; +import java.util.Map; + +import static org.opensearch.dataprepper.plugins.source.atlassian.utils.Constants.MAX_RETRIES; +import static org.opensearch.dataprepper.plugins.source.atlassian.utils.Constants.SLASH; + +/** + * The type Jira service. + */ + +public class AtlassianOauthConfig implements AtlassianAuthConfig { + + public static final String OAuth2_URL = "https://api.atlassian.com/ex/jira/"; + public static final String ACCESSIBLE_RESOURCES = "https://api.atlassian.com/oauth/token/accessible-resources"; + public static final String TOKEN_LOCATION = "https://auth.atlassian.com/oauth/token"; + + public static final String EXPIRES_IN = "expires_in"; + public static final String REFRESH_TOKEN = "refresh_token"; + public static final String ACCESS_TOKEN = "access_token"; + private static final Logger log = LoggerFactory.getLogger(AtlassianOauthConfig.class); + RestTemplate restTemplate = new RestTemplate(); + private String url; + @Getter + private int expiresInSeconds = 0; + @Getter + private Instant expireTime = Instant.ofEpochMilli(0); + @Getter + private String accessToken; + @Getter + private String refreshToken; + private String cloudId = null; + private final String clientId; + private final String clientSecret; + private final AtlassianSourceConfig confluenceSourceConfig; + private final Object cloudIdFetchLock = new Object(); + private final Object tokenRenewLock = new Object(); + + public AtlassianOauthConfig(AtlassianSourceConfig confluenceSourceConfig) { + this.confluenceSourceConfig = confluenceSourceConfig; + this.accessToken = (String) confluenceSourceConfig.getAuthenticationConfig().getOauth2Config() + .getAccessToken().getValue(); + this.refreshToken = (String) confluenceSourceConfig.getAuthenticationConfig() + .getOauth2Config().getRefreshToken().getValue(); + this.clientId = confluenceSourceConfig.getAuthenticationConfig().getOauth2Config().getClientId(); + this.clientSecret = confluenceSourceConfig.getAuthenticationConfig().getOauth2Config().getClientSecret(); + } + + public String getJiraAccountCloudId() { + log.info("Getting Jira Account Cloud ID"); + synchronized (cloudIdFetchLock) { + if (this.cloudId != null) { + //Someone else must have initialized it + return this.cloudId; + } + + int retryCount = 0; + while (retryCount < MAX_RETRIES) { + retryCount++; + try { + HttpHeaders headers = new HttpHeaders(); + headers.setBearerAuth(accessToken); + headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED); + HttpEntity entity = new HttpEntity<>(headers); + ResponseEntity exchangeResponse = + restTemplate.exchange(ACCESSIBLE_RESOURCES, HttpMethod.GET, entity, Object.class); + List> listResponse = (List>) exchangeResponse.getBody(); + Map response = listResponse.get(0); + this.cloudId = (String) response.get("id"); + return this.cloudId; + } catch (HttpClientErrorException e) { + if (e.getRawStatusCode() == HttpStatus.UNAUTHORIZED.value()) { + renewCredentials(); + } + log.error("Error occurred while accessing resources: ", e); + } + } + throw new UnauthorizedException(String.format("Access token expired. Unable to renew even after %s attempts", MAX_RETRIES)); + } + } + + public void renewCredentials() { + Instant currentTime = Instant.now(); + if (expireTime.isAfter(currentTime)) { + //There is still time to renew or someone else must have already renewed it + return; + } + + synchronized (tokenRenewLock) { + if (expireTime.isAfter(currentTime)) { + //Someone else must have already renewed it + return; + } + + log.info("Renewing access token and refresh token pair for Jira Connector."); + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + String payloadTemplate = "{\"grant_type\": \"%s\", \"client_id\": \"%s\", \"client_secret\": \"%s\", \"refresh_token\": \"%s\"}"; + String payload = String.format(payloadTemplate, "refresh_token", clientId, clientSecret, refreshToken); + HttpEntity entity = new HttpEntity<>(payload, headers); + + Oauth2Config oauth2Config = confluenceSourceConfig.getAuthenticationConfig().getOauth2Config(); + try { + ResponseEntity responseEntity = restTemplate.postForEntity(TOKEN_LOCATION, entity, Map.class); + Map oauthClientResponse = responseEntity.getBody(); + this.accessToken = (String) oauthClientResponse.get(ACCESS_TOKEN); + this.refreshToken = (String) oauthClientResponse.get(REFRESH_TOKEN); + this.expiresInSeconds = (int) oauthClientResponse.get(EXPIRES_IN); + this.expireTime = Instant.now().plusSeconds(expiresInSeconds); + // updating config object's PluginConfigVariable so that it updates the underlying Secret store + oauth2Config.getAccessToken().setValue(this.accessToken); + oauth2Config.getRefreshToken().setValue(this.refreshToken); + log.info("Access Token and Refresh Token pair is now refreshed. Corresponding Secret store key updated."); + } catch (HttpClientErrorException ex) { + this.expireTime = Instant.ofEpochMilli(0); + this.expiresInSeconds = 0; + HttpStatus statusCode = ex.getStatusCode(); + log.error("Failed to renew access token. Status code: {}, Error Message: {}", + statusCode, ex.getMessage()); + if (statusCode == HttpStatus.FORBIDDEN || statusCode == HttpStatus.UNAUTHORIZED) { + log.info("Trying to refresh the secrets"); + // Refreshing the secrets. It should help if someone already renewed the tokens. + // Refreshing one of the secret refreshes the entire store so triggering refresh on just one + oauth2Config.getAccessToken().refresh(); + this.accessToken = (String) oauth2Config.getAccessToken().getValue(); + this.refreshToken = (String) oauth2Config.getRefreshToken().getValue(); + this.expireTime = Instant.now().plusSeconds(10); + } + throw new RuntimeException("Failed to renew access token message:" + ex.getMessage(), ex); + } + } + } + + @Override + public String getUrl() { + if (!StringUtils.hasLength(url)) { + synchronized (cloudIdFetchLock) { + if (!StringUtils.hasLength(url)) { + initCredentials(); + } + } + } + return url; + } + + /** + * Method for getting Jira url based on auth type. + */ + @Override + public void initCredentials() { + //For OAuth based flow, we use a different Jira url + this.cloudId = getJiraAccountCloudId(); + this.url = OAuth2_URL + this.cloudId + SLASH; + } +} diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/utils/Constants.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/utils/Constants.java new file mode 100644 index 0000000000..17d702d201 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/main/java/org/opensearch/dataprepper/plugins/source/atlassian/utils/Constants.java @@ -0,0 +1,23 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.atlassian.utils; + +/** + * The type Constants. + */ +public class Constants { + + public static final int MAX_RETRIES = 6; + + public static final String OAUTH2 = "OAuth2"; + public static final String BASIC = "Basic"; + public static final String SLASH = "/"; +} \ No newline at end of file diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/AtlassianRestClientTest.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/AtlassianRestClientTest.java new file mode 100644 index 0000000000..bb1357dd3e --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/AtlassianRestClientTest.java @@ -0,0 +1,75 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.atlassian.rest; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.plugins.source.atlassian.rest.auth.AtlassianAuthConfig; +import org.opensearch.dataprepper.plugins.source.source_crawler.exception.UnauthorizedException; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.util.UriComponentsBuilder; + +import java.net.URI; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +public class AtlassianRestClientTest { + + @Mock + private RestTemplate restTemplate; + + @Mock + private AtlassianAuthConfig authConfig; + + private static Stream provideHttpStatusCodesWithExceptionClass() { + return Stream.of( + Arguments.of(HttpStatus.FORBIDDEN, UnauthorizedException.class), + Arguments.of(HttpStatus.UNAUTHORIZED, RuntimeException.class), + Arguments.of(HttpStatus.TOO_MANY_REQUESTS, RuntimeException.class), + Arguments.of(HttpStatus.INSUFFICIENT_STORAGE, RuntimeException.class) + ); + } + + @ParameterizedTest + @MethodSource("provideHttpStatusCodesWithExceptionClass") + void testInvokeRestApiTokenExpired(HttpStatus statusCode, Class expectedExceptionType) { + AtlassianRestClient atlassianRestClient = new AtlassianRestClient(restTemplate, authConfig); + atlassianRestClient.setSleepTimeMultiplier(1); + when(restTemplate.getForEntity(any(URI.class), any(Class.class))).thenThrow(new HttpClientErrorException(statusCode)); + URI uri = UriComponentsBuilder.fromHttpUrl("https://example.com/rest/api/2/issue/key").buildAndExpand().toUri(); + assertThrows(expectedExceptionType, () -> atlassianRestClient.invokeRestApi(uri, Object.class)); + } + + @Test + void testInvokeRestApiSuccessFullResponse() { + AtlassianRestClient atlassianRestClient = new AtlassianRestClient(restTemplate, authConfig); + atlassianRestClient.setSleepTimeMultiplier(1); + String apiReponse = "{\"api-response\":\"ok\"}"; + ResponseEntity responseEntity = new ResponseEntity(apiReponse, HttpStatus.OK); + when(restTemplate.getForEntity(any(URI.class), any(Class.class))).thenReturn(responseEntity); + URI uri = UriComponentsBuilder.fromHttpUrl("https://example.com/rest/api/2/issue/key").buildAndExpand().toUri(); + assertEquals(apiReponse, atlassianRestClient.invokeRestApi(uri, String.class).getBody()); + } + +} diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/BasicAuthInterceptorTest.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/BasicAuthInterceptorTest.java new file mode 100644 index 0000000000..b88d887e42 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/BasicAuthInterceptorTest.java @@ -0,0 +1,79 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.atlassian.rest; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.configuration.AuthenticationConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.configuration.BasicConfig; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpRequest; +import org.springframework.http.client.ClientHttpRequestExecution; +import org.springframework.http.client.ClientHttpResponse; + +import java.io.IOException; +import java.util.Base64; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +public class BasicAuthInterceptorTest { + + @Mock + AuthenticationConfig authenticationConfig; + @Mock + BasicConfig basicConfig; + @Mock + private HttpRequest mockRequest; + @Mock + private ClientHttpRequestExecution mockExecution; + @Mock + private ClientHttpResponse mockResponse; + @Mock + private AtlassianSourceConfig mockConfig; + @Mock + private HttpHeaders mockHeaders; + + private BasicAuthInterceptor interceptor; + + @BeforeEach + void setUp() { + when(mockConfig.getAuthenticationConfig()).thenReturn(authenticationConfig); + when(authenticationConfig.getBasicConfig()).thenReturn(basicConfig); + when(basicConfig.getUsername()).thenReturn("testUser"); + when(basicConfig.getPassword()).thenReturn("testPassword"); + when(mockRequest.getHeaders()).thenReturn(mockHeaders); + interceptor = new BasicAuthInterceptor(mockConfig); + } + + @Test + void testInterceptAddsAuthorizationHeader() throws IOException { + when(mockExecution.execute(any(HttpRequest.class), any(byte[].class))).thenReturn(mockResponse); + + ClientHttpResponse response = interceptor.intercept(mockRequest, new byte[0], mockExecution); + + verify(mockHeaders).set(eq(HttpHeaders.AUTHORIZATION), argThat(value -> + value.startsWith("Basic ") && + new String(Base64.getDecoder().decode(value.substring(6))).equals("testUser:testPassword") + )); + assertEquals(mockResponse, response); + } + +} diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/CustomRestTemplateConfigTest.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/CustomRestTemplateConfigTest.java new file mode 100644 index 0000000000..0e52d16eb6 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/CustomRestTemplateConfigTest.java @@ -0,0 +1,104 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.atlassian.rest; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.model.plugin.PluginConfigVariable; +import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.configuration.AuthenticationConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.configuration.BasicConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.configuration.Oauth2Config; +import org.opensearch.dataprepper.plugins.source.atlassian.rest.auth.AtlassianAuthConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.utils.Constants; +import org.springframework.http.client.ClientHttpRequestInterceptor; +import org.springframework.http.client.InterceptingClientHttpRequestFactory; +import org.springframework.web.client.RestTemplate; + +import java.util.List; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class CustomRestTemplateConfigTest { + + private CustomRestTemplateConfig config; + + @Mock + private AtlassianSourceConfig mockSourceConfig; + + @Mock + private AtlassianAuthConfig mockAuthConfig; + + @Mock + private BasicConfig mockBasicConfig; + + @Mock + private Oauth2Config mockOauth2Config; + + @Mock + private PluginConfigVariable accessTokenPluginConfigVariable; + + @Mock + private PluginConfigVariable refreshTokenPluginConfigVariable; + + @Mock + private AuthenticationConfig mockAuthenticationConfig; + + private static Stream provideAuthTypeAndExpectedInterceptorType() { + return Stream.of( + Arguments.of(Constants.OAUTH2, OAuth2RequestInterceptor.class), + Arguments.of(Constants.BASIC, BasicAuthInterceptor.class), + Arguments.of("Default", BasicAuthInterceptor.class), + Arguments.of(null, BasicAuthInterceptor.class) + ); + } + + @BeforeEach + void setUp() { + config = new CustomRestTemplateConfig(); + } + + @ParameterizedTest + @MethodSource("provideAuthTypeAndExpectedInterceptorType") + void testBasicAuthRestTemplateWithOAuth2(String authType, Class interceptorClassType) { + when(mockSourceConfig.getAuthType()).thenReturn(authType); + lenient().when(mockSourceConfig.getAuthenticationConfig()).thenReturn(mockAuthenticationConfig); + lenient().when(mockAuthenticationConfig.getOauth2Config()).thenReturn(mockOauth2Config); + lenient().when(mockOauth2Config.getAccessToken()).thenReturn(accessTokenPluginConfigVariable); + lenient().when(mockOauth2Config.getRefreshToken()).thenReturn(refreshTokenPluginConfigVariable); + lenient().when(accessTokenPluginConfigVariable.getValue()).thenReturn("accessToken"); + lenient().when(mockOauth2Config.getClientId()).thenReturn("clientId"); + lenient().when(mockOauth2Config.getClientSecret()).thenReturn("clientSecret"); + lenient().when(mockAuthenticationConfig.getBasicConfig()).thenReturn(mockBasicConfig); + lenient().when(mockBasicConfig.getUsername()).thenReturn("username"); + lenient().when(mockBasicConfig.getPassword()).thenReturn("password"); + + RestTemplate restTemplate = config.basicAuthRestTemplate(mockSourceConfig, mockAuthConfig); + assertNotNull(restTemplate); + assertInstanceOf(InterceptingClientHttpRequestFactory.class, restTemplate.getRequestFactory()); + List interceptors = restTemplate.getInterceptors(); + assertEquals(1, interceptors.size()); + assertInstanceOf(interceptorClassType, interceptors.get(0)); + } + +} + diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/OAuth2RequestInterceptorTest.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/OAuth2RequestInterceptorTest.java new file mode 100644 index 0000000000..b1b6f51b3f --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/OAuth2RequestInterceptorTest.java @@ -0,0 +1,66 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.atlassian.rest; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.plugins.source.atlassian.rest.auth.AtlassianOauthConfig; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpRequest; +import org.springframework.http.client.ClientHttpRequestExecution; +import org.springframework.http.client.ClientHttpResponse; + +import java.io.IOException; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +public class OAuth2RequestInterceptorTest { + + @Mock + private HttpRequest mockRequest; + + @Mock + private ClientHttpRequestExecution mockExecution; + + @Mock + private ClientHttpResponse mockResponse; + + @Mock + private AtlassianOauthConfig mockConfig; + + @Mock + private HttpHeaders mockHeaders; + + private OAuth2RequestInterceptor interceptor; + + @BeforeEach + void setUp() { + when(mockConfig.getAccessToken()).thenReturn("testAccessToken"); + when(mockRequest.getHeaders()).thenReturn(mockHeaders); + interceptor = new OAuth2RequestInterceptor(mockConfig); + } + + + @Test + void testInterceptAddsAuthorizationHeader() throws IOException { + when(mockExecution.execute(any(HttpRequest.class), any(byte[].class))).thenReturn(mockResponse); + ClientHttpResponse response = interceptor.intercept(mockRequest, new byte[0], mockExecution); + verify(mockHeaders).setBearerAuth("testAccessToken"); + assertEquals(mockResponse, response); + } +} diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianAuthFactoryTest.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianAuthFactoryTest.java new file mode 100644 index 0000000000..7dc618f744 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianAuthFactoryTest.java @@ -0,0 +1,74 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.atlassian.rest.auth; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.model.plugin.PluginConfigVariable; +import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.configuration.AuthenticationConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.configuration.Oauth2Config; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.mockito.Mockito.when; +import static org.opensearch.dataprepper.plugins.source.atlassian.utils.Constants.OAUTH2; + +@ExtendWith(MockitoExtension.class) +public class AtlassianAuthFactoryTest { + + @Mock + private AtlassianSourceConfig sourceConfig; + + @Mock + private AuthenticationConfig authenticationConfig; + + @Mock + private Oauth2Config oauth2Config; + + @Mock + private PluginConfigVariable accessTokenPluginConfigVariable; + + @Mock + private PluginConfigVariable refreshTokenPluginConfigVariable; + + private AtlassianAuthFactory confluenceAuthFactory; + + @BeforeEach + void setUp() { + confluenceAuthFactory = new AtlassianAuthFactory(sourceConfig); + } + + @Test + void testGetObjectOauth2() { + when(sourceConfig.getAuthType()).thenReturn(OAUTH2); + when(sourceConfig.getAuthenticationConfig()).thenReturn(authenticationConfig); + when(authenticationConfig.getOauth2Config()).thenReturn(oauth2Config); + when(oauth2Config.getRefreshToken()).thenReturn(refreshTokenPluginConfigVariable); + when(oauth2Config.getAccessToken()).thenReturn(accessTokenPluginConfigVariable); + when(accessTokenPluginConfigVariable.getValue()).thenReturn("mockRefreshToken"); + assertInstanceOf(AtlassianOauthConfig.class, confluenceAuthFactory.getObject()); + } + + @Test + void testGetObjectBasicAuth() { + when(sourceConfig.getAccountUrl()).thenReturn("https://example.com"); + assertInstanceOf(AtlassianBasicAuthConfig.class, confluenceAuthFactory.getObject()); + } + + @Test + void testGetObjectType() { + assertEquals(AtlassianAuthConfig.class, confluenceAuthFactory.getObjectType()); + } +} diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianBasicAuthConfigTest.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianBasicAuthConfigTest.java new file mode 100644 index 0000000000..7c0a8196af --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianBasicAuthConfigTest.java @@ -0,0 +1,48 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.atlassian.rest.auth; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +public class AtlassianBasicAuthConfigTest { + + String url = "https://example.com"; + @Mock + private AtlassianSourceConfig confluenceSourceConfig; + private AtlassianBasicAuthConfig jiraBasicAuthConfig; + + @BeforeEach + void setUp() { + when(confluenceSourceConfig.getAccountUrl()).thenReturn(url); + jiraBasicAuthConfig = new AtlassianBasicAuthConfig(confluenceSourceConfig); + } + + @Test + void testGetUrl() { + assertEquals(jiraBasicAuthConfig.getUrl(), url + '/'); + + } + + @Test + void DoNothingForBasicAuthentication() { + jiraBasicAuthConfig.initCredentials(); + jiraBasicAuthConfig.renewCredentials(); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianOauthConfigTest.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianOauthConfigTest.java new file mode 100644 index 0000000000..640d6652be --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/rest/auth/AtlassianOauthConfigTest.java @@ -0,0 +1,187 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.atlassian.rest.auth; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.model.plugin.PluginConfigVariable; +import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.configuration.Oauth2Config; +import org.opensearch.dataprepper.plugins.source.atlassian.utils.ConfigUtilForTests; +import org.opensearch.dataprepper.plugins.source.source_crawler.exception.UnauthorizedException; +import org.opensearch.dataprepper.test.helper.ReflectivelySetField; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.RestTemplate; + +import java.time.Instant; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.opensearch.dataprepper.plugins.source.atlassian.utils.Constants.MAX_RETRIES; + +@ExtendWith(MockitoExtension.class) +public class AtlassianOauthConfigTest { + + @Mock + RestTemplate restTemplateMock; + + + AtlassianSourceConfig confluenceSourceConfig; + + @Mock + PluginConfigVariable accessTokenVariable; + + @BeforeEach + void setUp() { + confluenceSourceConfig = ConfigUtilForTests.createJiraConfigurationFromYaml("oauth2-auth-jira-pipeline.yaml"); + } + + @Test + void testRenewToken() throws InterruptedException { + Instant testStartTime = Instant.now(); + Map firstMockResponseMap = Map.of("access_token", "first_mock_access_token", + "refresh_token", "first_mock_refresh_token", + "expires_in", 3600); + AtlassianOauthConfig jiraOauthConfig = new AtlassianOauthConfig(confluenceSourceConfig); + when(restTemplateMock.postForEntity(any(String.class), any(HttpEntity.class), any(Class.class))) + .thenReturn(new ResponseEntity<>(firstMockResponseMap, HttpStatus.OK)); + jiraOauthConfig.restTemplate = restTemplateMock; + ExecutorService executor = Executors.newFixedThreadPool(2); + Future firstCall = executor.submit(jiraOauthConfig::renewCredentials); + Future secondCall = executor.submit(jiraOauthConfig::renewCredentials); + while (!firstCall.isDone() || !secondCall.isDone()) { + // Do nothing. Wait for the calls to complete + Thread.sleep(10); + } + executor.shutdown(); + assertNotNull(jiraOauthConfig.getAccessToken()); + assertNotNull(jiraOauthConfig.getExpireTime()); + assertEquals(jiraOauthConfig.getRefreshToken(), "first_mock_refresh_token"); + assertEquals(jiraOauthConfig.getExpiresInSeconds(), 3600); + assertEquals(jiraOauthConfig.getAccessToken(), "first_mock_access_token"); + assertTrue(jiraOauthConfig.getExpireTime().isAfter(testStartTime)); + Instant expectedNewExpireTime = Instant.ofEpochMilli(testStartTime.toEpochMilli() + 3601 * 1000); + assertTrue(jiraOauthConfig.getExpireTime().isBefore(expectedNewExpireTime), + String.format("Expected that %s time should be before %s", jiraOauthConfig.getExpireTime(), expectedNewExpireTime)); + verify(restTemplateMock, times(1)).postForEntity(any(String.class), any(HttpEntity.class), any(Class.class)); + + } + + @Test + void testFailedToRenewAccessToken() throws NoSuchFieldException, IllegalAccessException { + AtlassianOauthConfig jiraOauthConfig = new AtlassianOauthConfig(confluenceSourceConfig); + Oauth2Config oauth2Config = confluenceSourceConfig.getAuthenticationConfig().getOauth2Config(); + ReflectivelySetField.setField(Oauth2Config.class, oauth2Config, "accessToken", accessTokenVariable); + when(restTemplateMock.postForEntity(any(String.class), any(HttpEntity.class), any(Class.class))) + .thenThrow(HttpClientErrorException.class); + jiraOauthConfig.restTemplate = restTemplateMock; + assertThrows(RuntimeException.class, jiraOauthConfig::renewCredentials); + verify(oauth2Config.getAccessToken(), times(0)) + .refresh(); + } + + @Test + void testFailedToRenewAccessToken_with_unauthorized_and_trigger_secrets_refresh() + throws NoSuchFieldException, IllegalAccessException { + AtlassianOauthConfig jiraOauthConfig = new AtlassianOauthConfig(confluenceSourceConfig); + Oauth2Config oauth2Config = confluenceSourceConfig.getAuthenticationConfig().getOauth2Config(); + ReflectivelySetField.setField(Oauth2Config.class, oauth2Config, "accessToken", accessTokenVariable); + HttpClientErrorException unAuthorizedException = new HttpClientErrorException(HttpStatus.UNAUTHORIZED); + when(restTemplateMock.postForEntity(any(String.class), any(HttpEntity.class), any(Class.class))) + .thenThrow(unAuthorizedException); + jiraOauthConfig.restTemplate = restTemplateMock; + assertThrows(RuntimeException.class, jiraOauthConfig::renewCredentials); + verify(oauth2Config.getAccessToken(), times(1)) + .refresh(); + } + + + @Test + void testGetJiraAccountCloudId() throws InterruptedException { + Map mockGetCallResponse = new HashMap<>(); + mockGetCallResponse.put("id", "test_cloud_id"); + when(restTemplateMock.exchange(any(String.class), any(HttpMethod.class), any(HttpEntity.class), any(Class.class))) + .thenReturn(new ResponseEntity<>(List.of(mockGetCallResponse), HttpStatus.OK)); + AtlassianOauthConfig jiraOauthConfig = new AtlassianOauthConfig(confluenceSourceConfig); + jiraOauthConfig.restTemplate = restTemplateMock; + + ExecutorService executor = Executors.newFixedThreadPool(2); + Future firstCall = executor.submit(jiraOauthConfig::getUrl); + Future secondCall = executor.submit(jiraOauthConfig::getUrl); + while (!firstCall.isDone() || !secondCall.isDone()) { + // Do nothing. Wait for the calls to complete + Thread.sleep(10); + } + executor.shutdown(); + + assertEquals("test_cloud_id", jiraOauthConfig.getJiraAccountCloudId()); + assertEquals("https://api.atlassian.com/ex/jira/test_cloud_id/", jiraOauthConfig.getUrl()); + //calling second time shouldn't trigger rest call + jiraOauthConfig.getUrl(); + verify(restTemplateMock, times(1)) + .exchange(any(String.class), any(HttpMethod.class), any(HttpEntity.class), any(Class.class)); + } + + @Test + void testGetJiraAccountCloudIdUnauthorizedCase() { + + when(restTemplateMock.exchange(any(String.class), any(HttpMethod.class), any(HttpEntity.class), any(Class.class))) + .thenThrow(new HttpClientErrorException(HttpStatus.UNAUTHORIZED)); + Map mockRenewTokenResponse = Map.of("access_token", "first_mock_access_token", + "refresh_token", "first_mock_refresh_token", + "expires_in", 3600); + when(restTemplateMock.postForEntity(any(String.class), any(HttpEntity.class), any(Class.class))) + .thenReturn(new ResponseEntity<>(mockRenewTokenResponse, HttpStatus.OK)); + AtlassianOauthConfig jiraOauthConfig = new AtlassianOauthConfig(confluenceSourceConfig); + jiraOauthConfig.restTemplate = restTemplateMock; + + + assertThrows(UnauthorizedException.class, () -> jiraOauthConfig.initCredentials()); + verify(restTemplateMock, times(6)) + .exchange(any(String.class), any(HttpMethod.class), any(HttpEntity.class), any(Class.class)); + verify(restTemplateMock, times(1)) + .postForEntity(any(String.class), any(HttpEntity.class), any(Class.class)); + + } + + @Test + void testFailedToGetCloudId() { + when(restTemplateMock.exchange(any(String.class), any(HttpMethod.class), any(HttpEntity.class), any(Class.class))) + .thenThrow(new HttpClientErrorException(HttpStatus.UNAUTHORIZED)) + .thenThrow(HttpClientErrorException.class); + AtlassianOauthConfig jiraOauthConfig = new AtlassianOauthConfig(confluenceSourceConfig); + jiraOauthConfig.restTemplate = restTemplateMock; + assertThrows(RuntimeException.class, jiraOauthConfig::getUrl); + for (int i = 0; i <= MAX_RETRIES; i++) { + assertThrows(RuntimeException.class, jiraOauthConfig::getUrl); + } + } + +} diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/utils/ConfigUtilForTests.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/utils/ConfigUtilForTests.java new file mode 100644 index 0000000000..61d1889935 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/utils/ConfigUtilForTests.java @@ -0,0 +1,45 @@ +package org.opensearch.dataprepper.plugins.source.atlassian.utils; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.configuration.Oauth2Config; +import org.opensearch.dataprepper.test.helper.ReflectivelySetField; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.InputStream; + +public class ConfigUtilForTests { + + public static final Logger log = LoggerFactory.getLogger(ConfigUtilForTests.class); + + private static InputStream getResourceAsStream(String resourceName) { + InputStream inputStream = Thread.currentThread().getContextClassLoader().getResourceAsStream(resourceName); + if (inputStream == null) { + inputStream = ConfigUtilForTests.class.getResourceAsStream("/" + resourceName); + } + return inputStream; + } + + public static AtlassianSourceConfig createJiraConfigurationFromYaml(String fileName) { + ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory()); + try (InputStream inputStream = getResourceAsStream(fileName)) { + AtlassianSourceConfig confluenceSourceConfig = objectMapper.readValue(inputStream, AtlassianSourceConfig.class); + Oauth2Config oauth2Config = confluenceSourceConfig.getAuthenticationConfig().getOauth2Config(); + if (oauth2Config != null) { + ReflectivelySetField.setField(Oauth2Config.class, oauth2Config, "accessToken", + new MockPluginConfigVariableImpl("mockAccessToken")); + ReflectivelySetField.setField(Oauth2Config.class, oauth2Config, "refreshToken", + new MockPluginConfigVariableImpl("mockRefreshToken")); + } + return confluenceSourceConfig; + } catch (IOException ex) { + log.error("Failed to parse pipeline Yaml", ex); + } catch (Exception e) { + throw new RuntimeException(e); + } + return null; + } +} diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/utils/MockPluginConfigVariableImpl.java b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/utils/MockPluginConfigVariableImpl.java new file mode 100644 index 0000000000..4c5f4e8e93 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/java/org/opensearch/dataprepper/plugins/source/atlassian/utils/MockPluginConfigVariableImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.atlassian.utils; + +import org.opensearch.dataprepper.model.plugin.PluginConfigVariable; + +/** + * Mock implementation of PluginConfigVariable interface used only for Unit Testing. + */ +public class MockPluginConfigVariableImpl implements PluginConfigVariable { + + private Object defaultValue; + + public MockPluginConfigVariableImpl(Object defaultValue) { + this.defaultValue = defaultValue; + } + + @Override + public Object getValue() { + return null; + } + + @Override + public void setValue(Object someValue) { + this.defaultValue = someValue; + } + + @Override + public void refresh() { + } + + @Override + public boolean isUpdatable() { + return true; + } +} diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/resources/basic-auth-jira-pipeline.yaml b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/resources/basic-auth-jira-pipeline.yaml new file mode 100644 index 0000000000..0bfa6384e8 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/resources/basic-auth-jira-pipeline.yaml @@ -0,0 +1,6 @@ +hosts: ["https://jira.com/"] +authentication: + basic: + username: "jiraId" + password: "jiraApiKey" + diff --git a/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/resources/oauth2-auth-jira-pipeline.yaml b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/resources/oauth2-auth-jira-pipeline.yaml new file mode 100644 index 0000000000..7a4afd3abf --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/atlassian-commons/src/test/resources/oauth2-auth-jira-pipeline.yaml @@ -0,0 +1,6 @@ +hosts: [ "https://jira.com/" ] +authentication: + oauth2: + client_id: "client_id" + client_secret: "client_secret" + diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/README.md b/data-prepper-plugins/saas-source-plugins/confluence-source/README.md new file mode 100644 index 0000000000..a5f5d962c1 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/README.md @@ -0,0 +1,9 @@ +# Metrics + +### Counter + +- `issuesRequested`: measures total number of issue Requests sent. + +### Timer + +- `requestProcessDuration`: measures latency of requests processed by the jira source plugin. diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/build.gradle b/data-prepper-plugins/saas-source-plugins/confluence-source/build.gradle new file mode 100644 index 0000000000..b627b772dd --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/build.gradle @@ -0,0 +1,31 @@ +plugins { + id 'java' +} + +dependencies { + + implementation project(path: ':data-prepper-plugins:saas-source-plugins:source-crawler') + implementation project(path: ':data-prepper-plugins:saas-source-plugins:atlassian-commons') + implementation project(path: ':data-prepper-api') + implementation project(path: ':data-prepper-plugins:aws-plugin-api') + implementation project(path: ':data-prepper-plugins:buffer-common') + implementation project(path: ':data-prepper-plugins:common') + + implementation libs.commons.io + implementation 'io.micrometer:micrometer-core' + implementation 'com.fasterxml.jackson.core:jackson-core' + implementation 'com.fasterxml.jackson.core:jackson-databind' + implementation 'javax.inject:javax.inject:1' + implementation 'org.jsoup:jsoup:1.18.3' + + implementation 'org.projectlombok:lombok:1.18.30' + annotationProcessor 'org.projectlombok:lombok:1.18.30' + + testImplementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.4' + testImplementation project(path: ':data-prepper-test-common') + + implementation(libs.spring.context) { + exclude group: 'commons-logging', module: 'commons-logging' + } + implementation(libs.spring.web) +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceClient.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceClient.java new file mode 100644 index 0000000000..225a9721f6 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceClient.java @@ -0,0 +1,149 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.annotations.VisibleForTesting; +import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSet; +import org.opensearch.dataprepper.model.buffer.Buffer; +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.event.EventType; +import org.opensearch.dataprepper.model.event.JacksonEvent; +import org.opensearch.dataprepper.model.record.Record; +import org.opensearch.dataprepper.plugins.source.confluence.utils.HtmlToTextConversionUtil; +import org.opensearch.dataprepper.plugins.source.source_crawler.base.CrawlerClient; +import org.opensearch.dataprepper.plugins.source.source_crawler.base.CrawlerSourceConfig; +import org.opensearch.dataprepper.plugins.source.source_crawler.base.PluginExecutorServiceProvider; +import org.opensearch.dataprepper.plugins.source.source_crawler.coordination.state.SaasWorkerProgressState; +import org.opensearch.dataprepper.plugins.source.source_crawler.model.ItemInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Named; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import static java.util.concurrent.CompletableFuture.supplyAsync; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.SPACE; + +/** + * This class represents a Confluence client. + */ +@Named +public class ConfluenceClient implements CrawlerClient { + + private static final Logger log = LoggerFactory.getLogger(ConfluenceClient.class); + private ObjectMapper objectMapper = new ObjectMapper(); + private Instant lastPollTime; + private final ConfluenceService service; + private final ConfluenceIterator confluenceIterator; + private final ExecutorService executorService; + private final CrawlerSourceConfig configuration; + private final int bufferWriteTimeoutInSeconds = 10; + + public ConfluenceClient(ConfluenceService service, + ConfluenceIterator confluenceIterator, + PluginExecutorServiceProvider executorServiceProvider, + ConfluenceSourceConfig sourceConfig) { + this.service = service; + this.confluenceIterator = confluenceIterator; + this.executorService = executorServiceProvider.get(); + this.configuration = sourceConfig; + } + + @Override + public Iterator listItems() { + confluenceIterator.initialize(lastPollTime); + return confluenceIterator; + } + + @Override + public void setLastPollTime(Instant lastPollTime) { + log.trace("Setting the lastPollTime: {}", lastPollTime); + this.lastPollTime = lastPollTime; + } + + @VisibleForTesting + public void injectObjectMapper(ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + } + + @Override + public void executePartition(SaasWorkerProgressState state, + Buffer> buffer, + AcknowledgementSet acknowledgementSet) { + log.trace("Executing the partition: {} with {} ticket(s)", + state.getKeyAttributes(), state.getItemIds().size()); + List itemIds = state.getItemIds(); + Map keyAttributes = state.getKeyAttributes(); + String space = (String) keyAttributes.get(SPACE); + Instant eventTime = state.getExportStartTime(); + List itemInfos = new ArrayList<>(); + for (String itemId : itemIds) { + if (itemId == null) { + continue; + } + ItemInfo itemInfo = ConfluenceItemInfo.builder() + .withItemId(itemId) + .withId(itemId) + .withSpace(space) + .withEventTime(eventTime) + .withMetadata(keyAttributes).build(); + itemInfos.add(itemInfo); + } + + String eventType = EventType.DOCUMENT.toString(); + List> recordsToWrite = itemInfos + .parallelStream() + .map(t -> (Supplier) (() -> service.getContent(t.getId()))) + .map(supplier -> supplyAsync(supplier, this.executorService)) + .map(CompletableFuture::join) + .map(contentJson -> { + try { + ObjectNode contentJsonObj = objectMapper.readValue(contentJson, new TypeReference<>() { + }); + return HtmlToTextConversionUtil.convertHtmlToText(contentJsonObj, "body/view/value"); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + }) + .map(t -> (Event) JacksonEvent.builder() + .withEventType(eventType) + .withData(t) + .build()) + .map(Record::new) + .collect(Collectors.toList()); + + try { + if (configuration.isAcknowledgments()) { + recordsToWrite.forEach(eventRecord -> acknowledgementSet.add(eventRecord.getData())); + buffer.writeAll(recordsToWrite, (int) Duration.ofSeconds(bufferWriteTimeoutInSeconds).toMillis()); + acknowledgementSet.complete(); + } else { + buffer.writeAll(recordsToWrite, (int) Duration.ofSeconds(bufferWriteTimeoutInSeconds).toMillis()); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + + } +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceItemInfo.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceItemInfo.java new file mode 100644 index 0000000000..b1861b5b69 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceItemInfo.java @@ -0,0 +1,158 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence; + +import lombok.Getter; +import lombok.Setter; +import org.opensearch.dataprepper.plugins.source.confluence.models.ConfluenceItem; +import org.opensearch.dataprepper.plugins.source.confluence.utils.ConfluenceContentType; +import org.opensearch.dataprepper.plugins.source.confluence.utils.Constants; +import org.opensearch.dataprepper.plugins.source.source_crawler.model.ItemInfo; + +import java.time.Instant; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.CONTENT_ID; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.CONTENT_TITLE; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.CREATED; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.LAST_MODIFIED; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.SPACE_KEY; + +@Setter +@Getter +public class ConfluenceItemInfo implements ItemInfo { + private String space; + // either a page or a BlogPost + private String contentType; + private String id; + private String itemId; + private Map metadata; + private Instant eventTime; + + public ConfluenceItemInfo(String id, + String itemId, + String space, + String contentType, + Map metadata, + Instant eventTime + ) { + this.id = id; + this.space = space; + this.contentType = contentType; + this.itemId = itemId; + this.metadata = metadata; + this.eventTime = eventTime; + } + + public static ConfluenceItemInfoBuilder builder() { + return new ConfluenceItemInfoBuilder(); + } + + @Override + public String getPartitionKey() { + return space + "|" + contentType + "|" + UUID.randomUUID(); + } + + @Override + public String getId() { + return id; + } + + @Override + public Map getKeyAttributes() { + return Map.of(Constants.SPACE, space); + } + + @Override + public Instant getLastModifiedAt() { + long updatedAtMillis = getMetadataField(Constants.LAST_MODIFIED); + long createdAtMillis = getMetadataField(Constants.CREATED); + return createdAtMillis > updatedAtMillis ? + Instant.ofEpochMilli(createdAtMillis) : Instant.ofEpochMilli(updatedAtMillis); + } + + private Long getMetadataField(String fieldName) { + Object value = this.metadata.get(fieldName); + if (value == null) { + return 0L; + } else if (value instanceof Long) { + return (Long) value; + } else if (value instanceof String) { + try { + return Long.parseLong((String) value); + } catch (Exception e) { + return 0L; + } + } + return 0L; + } + + public static class ConfluenceItemInfoBuilder { + private Map metadata; + private Instant eventTime; + private String id; + private String itemId; + private String space; + private String contentType; + + public ConfluenceItemInfoBuilder() { + } + + public ConfluenceItemInfo build() { + return new ConfluenceItemInfo(id, itemId, space, contentType, metadata, eventTime); + } + + public ConfluenceItemInfoBuilder withMetadata(Map metadata) { + this.metadata = metadata; + return this; + } + + public ConfluenceItemInfoBuilder withEventTime(Instant eventTime) { + this.eventTime = eventTime; + return this; + } + + public ConfluenceItemInfoBuilder withItemId(String itemId) { + this.itemId = itemId; + return this; + } + + public ConfluenceItemInfoBuilder withId(String id) { + this.id = id; + return this; + } + + public ConfluenceItemInfoBuilder withSpace(String space) { + this.space = space; + return this; + } + + public ConfluenceItemInfoBuilder withContentBean(ConfluenceItem contentItem) { + Map contentItemMetadata = new HashMap<>(); + contentItemMetadata.put(SPACE_KEY, contentItem.getSpaceItem().getKey()); + contentItemMetadata.put(CONTENT_TITLE, contentItem.getTitle()); + contentItemMetadata.put(CREATED, contentItem.getCreatedTimeMillis()); + contentItemMetadata.put(LAST_MODIFIED, contentItem.getUpdatedTimeMillis()); + contentItemMetadata.put(CONTENT_ID, contentItem.getId()); + contentItemMetadata.put(ConfluenceService.CONTENT_TYPE, ConfluenceContentType.PAGE.getType()); + + this.space = contentItem.getSpaceItem().getKey(); + this.id = contentItem.getId(); + this.contentType = ConfluenceContentType.PAGE.getType(); + this.itemId = contentItem.getId(); + this.metadata = contentItemMetadata; + return this; + } + } + +} \ No newline at end of file diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceIterator.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceIterator.java new file mode 100644 index 0000000000..93ba62cec2 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceIterator.java @@ -0,0 +1,119 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence; + + +import com.google.common.annotations.VisibleForTesting; +import lombok.Setter; +import org.opensearch.dataprepper.plugins.source.source_crawler.base.PluginExecutorServiceProvider; +import org.opensearch.dataprepper.plugins.source.source_crawler.model.ItemInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Named; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; + +@Named +public class ConfluenceIterator implements Iterator { + + private static final int HAS_NEXT_TIMEOUT = 60; + private static final Logger log = LoggerFactory.getLogger(ConfluenceIterator.class); + @Setter + private long crawlerQWaitTimeMillis = 2000; + private Queue itemInfoQueue; + private Instant lastPollTime; + private boolean firstTime = true; + private final List> futureList = new ArrayList<>(); + private final ConfluenceSourceConfig sourceConfig; + private final ConfluenceService service; + private final ExecutorService crawlerTaskExecutor; + + public ConfluenceIterator(final ConfluenceService service, + PluginExecutorServiceProvider executorServiceProvider, + ConfluenceSourceConfig sourceConfig) { + this.service = service; + this.crawlerTaskExecutor = executorServiceProvider.get(); + this.sourceConfig = sourceConfig; + } + + @Override + public boolean hasNext() { + if (firstTime) { + log.trace("Crawling has been started"); + startCrawlerThreads(); + firstTime = false; + } + int timeout = HAS_NEXT_TIMEOUT; + while (isCrawlerRunning() && itemInfoQueue.isEmpty() && timeout > 0) { + try { + log.trace("Waiting for crawler queue to be filled for next {} seconds", timeout); + Thread.sleep(crawlerQWaitTimeMillis); + timeout--; + } catch (InterruptedException e) { + log.error("An exception has occurred while checking for the next document in crawling queue"); + Thread.currentThread().interrupt(); + } + } + return !this.itemInfoQueue.isEmpty(); + } + + private boolean isCrawlerRunning() { + boolean isRunning = false; + if (!futureList.isEmpty()) { + for (Future future : futureList) { + if (!future.isDone()) { + isRunning = true; + break; + } + } + } + return isRunning; + } + + private void startCrawlerThreads() { + futureList.add(crawlerTaskExecutor.submit(() -> + service.getPages(sourceConfig, lastPollTime, itemInfoQueue), false)); + } + + @Override + public ItemInfo next() { + if (hasNext()) { + return this.itemInfoQueue.remove(); + } else { + throw new NoSuchElementException(); + } + } + + /** + * Initialize. + * + * @param confluenceChangeLogToken the jira change log token + */ + public void initialize(Instant confluenceChangeLogToken) { + this.itemInfoQueue = new ConcurrentLinkedQueue<>(); + this.lastPollTime = confluenceChangeLogToken; + this.firstTime = true; + } + + @VisibleForTesting + public List> showFutureList() { + return futureList; + } + +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java new file mode 100644 index 0000000000..a95be6635b --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceService.java @@ -0,0 +1,259 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence; + +import io.micrometer.core.instrument.Counter; +import lombok.extern.slf4j.Slf4j; +import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.plugins.source.confluence.models.ConfluenceItem; +import org.opensearch.dataprepper.plugins.source.confluence.models.ConfluenceSearchResults; +import org.opensearch.dataprepper.plugins.source.confluence.rest.ConfluenceRestClient; +import org.opensearch.dataprepper.plugins.source.confluence.utils.ConfluenceConfigHelper; +import org.opensearch.dataprepper.plugins.source.confluence.utils.ConfluenceContentType; +import org.opensearch.dataprepper.plugins.source.source_crawler.exception.BadRequestException; +import org.opensearch.dataprepper.plugins.source.source_crawler.model.ItemInfo; +import org.springframework.util.CollectionUtils; + +import javax.inject.Named; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Queue; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.LAST_MODIFIED; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.CqlConstants.CLOSING_ROUND_BRACKET; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.CqlConstants.CONTENT_TYPE_IN; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.CqlConstants.CONTENT_TYPE_NOT_IN; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.CqlConstants.DELIMITER; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.CqlConstants.GREATER_THAN_EQUALS; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.CqlConstants.PREFIX; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.CqlConstants.SPACE_IN; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.CqlConstants.SPACE_NOT_IN; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.CqlConstants.SUFFIX; + + +/** + * Service class for interactive external Atlassian Confluence SaaS service and fetch required details using their rest apis. + */ + +@Slf4j +@Named +public class ConfluenceService { + + + public static final String CONTENT_TYPE = "ContentType"; + private static final String SEARCH_RESULTS_FOUND = "searchResultsFound"; + + private final ConfluenceSourceConfig confluenceSourceConfig; + private final ConfluenceRestClient confluenceRestClient; + private final Counter searchResultsFoundCounter; + + + public ConfluenceService(ConfluenceSourceConfig confluenceSourceConfig, + ConfluenceRestClient confluenceRestClient, + PluginMetrics pluginMetrics) { + this.confluenceSourceConfig = confluenceSourceConfig; + this.confluenceRestClient = confluenceRestClient; + this.searchResultsFoundCounter = pluginMetrics.counter(SEARCH_RESULTS_FOUND); + } + + /** + * Get Confluence entities. + * + * @param configuration the configuration. + * @param timestamp timestamp. + */ + public void getPages(ConfluenceSourceConfig configuration, Instant timestamp, Queue itemInfoQueue) { + log.trace("Started to fetch entities"); + searchForNewContentAndAddToQueue(configuration, timestamp, itemInfoQueue); + log.trace("Creating item information and adding in queue"); + } + + public String getContent(String contentId) { + return confluenceRestClient.getContent(contentId); + } + + /** + * Method for building Content Item Info. + * + * @param configuration Input Parameter + * @param timestamp Input Parameter + */ + private void searchForNewContentAndAddToQueue(ConfluenceSourceConfig configuration, Instant timestamp, + Queue itemInfoQueue) { + log.trace("Looking for Add/Modified tickets with a Search API call"); + StringBuilder cql = createContentFilterCriteria(configuration, timestamp); + int total; + int startAt = 0; + do { + ConfluenceSearchResults searchContentItems = confluenceRestClient.getAllContent(cql, startAt); + List contentList = new ArrayList<>(searchContentItems.getResults()); + total = searchContentItems.getSize(); + startAt += searchContentItems.getResults().size(); + addItemsToQueue(contentList, itemInfoQueue); + } while (startAt < total); + searchResultsFoundCounter.increment(total); + log.info("Number of content items found in search api call: {}", total); + } + + /** + * Add items to queue. + * + * @param contentList Content list. + * @param itemInfoQueue Item info queue. + */ + private void addItemsToQueue(List contentList, Queue itemInfoQueue) { + contentList.forEach(contentItem -> itemInfoQueue.add(ConfluenceItemInfo.builder() + .withEventTime(Instant.now()).withContentBean(contentItem).build())); + } + + + /** + * Method for creating Content Filter Criteria. + * + * @param configuration Input Parameter + * @param ts Input Parameter + * @return String Builder + */ + private StringBuilder createContentFilterCriteria(ConfluenceSourceConfig configuration, Instant ts) { + + log.info("Creating content filter criteria"); + if (!CollectionUtils.isEmpty(ConfluenceConfigHelper.getSpacesNameIncludeFilter(configuration)) || !CollectionUtils.isEmpty(ConfluenceConfigHelper.getSpacesNameExcludeFilter(configuration))) { + validateSpaceFilters(configuration); + } + + if (!CollectionUtils.isEmpty(ConfluenceConfigHelper.getContentTypeIncludeFilter(configuration)) || !CollectionUtils.isEmpty(ConfluenceConfigHelper.getContentTypeExcludeFilter(configuration))) { + validatePageTypeFilters(configuration); + } + + String formattedTimeStamp = LocalDateTime.ofInstant(ts, ZoneId.systemDefault()) + .format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm")); + StringBuilder cQl = new StringBuilder(LAST_MODIFIED + GREATER_THAN_EQUALS + "\"" + formattedTimeStamp + "\""); + if (!CollectionUtils.isEmpty(ConfluenceConfigHelper.getSpacesNameIncludeFilter(configuration))) { + cQl.append(SPACE_IN).append(ConfluenceConfigHelper.getSpacesNameIncludeFilter(configuration).stream() + .collect(Collectors.joining(DELIMITER, PREFIX, SUFFIX))) + .append(CLOSING_ROUND_BRACKET); + } + if (!CollectionUtils.isEmpty(ConfluenceConfigHelper.getSpacesNameExcludeFilter(configuration))) { + cQl.append(SPACE_NOT_IN).append(ConfluenceConfigHelper.getSpacesNameExcludeFilter(configuration).stream() + .collect(Collectors.joining(DELIMITER, PREFIX, SUFFIX))) + .append(CLOSING_ROUND_BRACKET); + } + if (!CollectionUtils.isEmpty(ConfluenceConfigHelper.getContentTypeIncludeFilter(configuration))) { + cQl.append(CONTENT_TYPE_IN).append(ConfluenceConfigHelper.getContentTypeIncludeFilter(configuration).stream() + .collect(Collectors.joining(DELIMITER, PREFIX, SUFFIX))) + .append(CLOSING_ROUND_BRACKET); + } + if (!CollectionUtils.isEmpty(ConfluenceConfigHelper.getContentTypeExcludeFilter(configuration))) { + cQl.append(CONTENT_TYPE_NOT_IN).append(ConfluenceConfigHelper.getContentTypeExcludeFilter(configuration).stream() + .collect(Collectors.joining(DELIMITER, PREFIX, SUFFIX))) + .append(CLOSING_ROUND_BRACKET); + } + + log.info("Created content filter criteria ConfluenceQl query: {}", cQl); + return cQl; + } + + /** + * Method for Validating Page Type Filters. + * + * @param configuration Input Parameter + */ + private void validatePageTypeFilters(ConfluenceSourceConfig configuration) { + log.trace("Validating Page Type filters"); + List badFilters = new ArrayList<>(); + Set includedPageType = new HashSet<>(); + List includedAndExcludedPageType = new ArrayList<>(); + ConfluenceConfigHelper.getContentTypeIncludeFilter(configuration).forEach(pageTypeFilter -> { + if (ConfluenceContentType.fromString(pageTypeFilter) == null) { + badFilters.add(pageTypeFilter); + } else { + includedPageType.add(pageTypeFilter); + } + }); + ConfluenceConfigHelper.getContentTypeExcludeFilter(configuration).forEach(pageTypeFilter -> { + if (includedPageType.contains(pageTypeFilter)) { + includedAndExcludedPageType.add(pageTypeFilter); + } + if (ConfluenceContentType.fromString(pageTypeFilter) == null) { + badFilters.add(pageTypeFilter); + } + }); + if (!badFilters.isEmpty()) { + String filters = String.join("\"" + badFilters + "\"", ", "); + log.error("One or more invalid Page Types found in filter configuration: {}", badFilters); + throw new BadRequestException("Bad request exception occurred " + + "Invalid Page Type key found in filter configuration " + + filters); + } + if (!includedAndExcludedPageType.isEmpty()) { + String filters = String.join("\"" + includedAndExcludedPageType + "\"", ", "); + log.error("One or more Page types found in both include and exclude: {}", includedAndExcludedPageType); + throw new BadRequestException("Bad request exception occurred " + + "Page Type filters is invalid because the following Page types are listed in both include and exclude" + + filters); + } + + } + + /** + * Method for Validating Space Filters. + * + * @param configuration Input Parameter + */ + private void validateSpaceFilters(ConfluenceSourceConfig configuration) { + log.trace("Validating space filters"); + List badFilters = new ArrayList<>(); + Set includedSpaces = new HashSet<>(); + List includedAndExcludedSpaces = new ArrayList<>(); + Pattern regex = Pattern.compile("[^A-Z0-9]"); + ConfluenceConfigHelper.getSpacesNameIncludeFilter(configuration).forEach(spaceFilter -> { + Matcher matcher = regex.matcher(spaceFilter); + includedSpaces.add(spaceFilter); + if (matcher.find() || spaceFilter.length() <= 1 || spaceFilter.length() > 10) { + badFilters.add(spaceFilter); + } + }); + ConfluenceConfigHelper.getSpacesNameExcludeFilter(configuration).forEach(spaceFilter -> { + Matcher matcher = regex.matcher(spaceFilter); + if (includedSpaces.contains(spaceFilter)) { + includedAndExcludedSpaces.add(spaceFilter); + } + if (matcher.find() || spaceFilter.length() <= 1 || spaceFilter.length() > 10) { + badFilters.add(spaceFilter); + } + }); + if (!badFilters.isEmpty()) { + String filters = String.join("\"" + badFilters + "\"", ", "); + log.error("One or more invalid Space keys found in filter configuration: {}", badFilters); + throw new BadRequestException("Bad request exception occurred " + + "Invalid Space key found in filter configuration for " + + filters); + } + if (!includedAndExcludedSpaces.isEmpty()) { + String filters = String.join("\"" + includedAndExcludedSpaces + "\"", ", "); + log.error("One or more Space keys found in both include and exclude: {}", includedAndExcludedSpaces); + throw new BadRequestException("Bad request exception occurred " + + "Space filters is invalid because the following space are listed in both include and exclude" + + filters); + } + + } + +} \ No newline at end of file diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSource.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSource.java new file mode 100644 index 0000000000..d126b0da10 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSource.java @@ -0,0 +1,79 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence; + + +import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSetManager; +import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; +import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; +import org.opensearch.dataprepper.model.buffer.Buffer; +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.plugin.PluginFactory; +import org.opensearch.dataprepper.model.record.Record; +import org.opensearch.dataprepper.model.source.Source; +import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.rest.auth.AtlassianAuthConfig; +import org.opensearch.dataprepper.plugins.source.confluence.utils.ConfluenceConfigHelper; +import org.opensearch.dataprepper.plugins.source.source_crawler.CrawlerApplicationContextMarker; +import org.opensearch.dataprepper.plugins.source.source_crawler.base.Crawler; +import org.opensearch.dataprepper.plugins.source.source_crawler.base.CrawlerSourcePlugin; +import org.opensearch.dataprepper.plugins.source.source_crawler.base.PluginExecutorServiceProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.PLUGIN_NAME; + + +/** + * Confluence connector entry point. + */ + +@DataPrepperPlugin(name = PLUGIN_NAME, + pluginType = Source.class, + pluginConfigurationType = ConfluenceSourceConfig.class, + packagesToScan = {CrawlerApplicationContextMarker.class, AtlassianSourceConfig.class, ConfluenceSource.class} +) +public class ConfluenceSource extends CrawlerSourcePlugin { + + private static final Logger log = LoggerFactory.getLogger(ConfluenceSource.class); + private final ConfluenceSourceConfig confluenceSourceConfig; + private final AtlassianAuthConfig jiraOauthConfig; + + @DataPrepperPluginConstructor + public ConfluenceSource(final PluginMetrics pluginMetrics, + final ConfluenceSourceConfig confluenceSourceConfig, + final AtlassianAuthConfig jiraOauthConfig, + final PluginFactory pluginFactory, + final AcknowledgementSetManager acknowledgementSetManager, + Crawler crawler, + PluginExecutorServiceProvider executorServiceProvider) { + super(PLUGIN_NAME, pluginMetrics, confluenceSourceConfig, pluginFactory, acknowledgementSetManager, crawler, executorServiceProvider); + log.info("Creating Confluence Source Plugin"); + this.confluenceSourceConfig = confluenceSourceConfig; + this.jiraOauthConfig = jiraOauthConfig; + } + + @Override + public void start(Buffer> buffer) { + log.info("Starting Confluence Source Plugin... "); + ConfluenceConfigHelper.validateConfig(confluenceSourceConfig); + jiraOauthConfig.initCredentials(); + super.start(buffer); + } + + @Override + public void stop() { + log.info("Stopping Confluence Source Plugin"); + super.stop(); + } + +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceConfig.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceConfig.java new file mode 100644 index 0000000000..b22ad46721 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceConfig.java @@ -0,0 +1,34 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Getter; +import org.opensearch.dataprepper.plugins.source.atlassian.AtlassianSourceConfig; +import org.opensearch.dataprepper.plugins.source.confluence.configuration.FilterConfig; +import org.opensearch.dataprepper.plugins.source.source_crawler.base.CrawlerSourceConfig; + +@Getter +public class ConfluenceSourceConfig extends AtlassianSourceConfig implements CrawlerSourceConfig { + /** + * Filter Config to filter what tickets get ingested + */ + @JsonProperty("filter") + private FilterConfig filterConfig; + + + /** + * Boolean property indicating end to end acknowledgments state + */ + @JsonProperty("acknowledgments") + private boolean acknowledgments = false; + +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/FilterConfig.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/FilterConfig.java new file mode 100644 index 0000000000..0ec468f2d8 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/FilterConfig.java @@ -0,0 +1,23 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.configuration; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Getter; + +@Getter +public class FilterConfig { + @JsonProperty("space") + private SpaceConfig spaceConfig; + + @JsonProperty("page_type") + private PageTypeConfig pageTypeConfig; +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/NameConfig.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/NameConfig.java new file mode 100644 index 0000000000..8e3a513d56 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/NameConfig.java @@ -0,0 +1,30 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + + +package org.opensearch.dataprepper.plugins.source.confluence.configuration; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.constraints.Size; +import lombok.Getter; + +import java.util.ArrayList; +import java.util.List; + +@Getter +public class NameConfig { + @JsonProperty("include") + @Size(max = 1000, message = "Space name type filter should not be more than 1000") + private List include = new ArrayList<>(); + + @JsonProperty("exclude") + @Size(max = 1000, message = "Space name type filter should not be more than 1000") + private List exclude = new ArrayList<>(); +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/PageTypeConfig.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/PageTypeConfig.java new file mode 100644 index 0000000000..756e9ac91b --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/PageTypeConfig.java @@ -0,0 +1,29 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.configuration; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.constraints.Size; +import lombok.Getter; + +import java.util.ArrayList; +import java.util.List; + +@Getter +public class PageTypeConfig { + @JsonProperty("include") + @Size(max = 1000, message = "Page type filter should not be more than 1000") + private List include = new ArrayList<>(); + + @JsonProperty("exclude") + @Size(max = 1000, message = "Page type filter should not be more than 1000") + private List exclude = new ArrayList<>(); +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/SpaceConfig.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/SpaceConfig.java new file mode 100644 index 0000000000..fe572011ff --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/configuration/SpaceConfig.java @@ -0,0 +1,20 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.configuration; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Getter; + +@Getter +public class SpaceConfig { + @JsonProperty("key") + private NameConfig nameConfig; +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceItem.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceItem.java new file mode 100644 index 0000000000..1806627793 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceItem.java @@ -0,0 +1,65 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.models; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Getter; +import lombok.Setter; + + +@Setter +@Getter +@JsonIgnoreProperties(ignoreUnknown = true) +public class ConfluenceItem { + + /** + * The ID of the issue. + */ + @JsonProperty("id") + private String id = null; + + /** + * The type of the issue. + */ + @JsonProperty("type") + private String type = null; + + /** + * The type of the issue. + */ + @JsonProperty("status") + private String status = null; + + /** + * The type of the issue. + */ + @JsonProperty("title") + private String title = null; + + /** + * Space this content belongs to + */ + @JsonProperty("space") + private SpaceItem spaceItem; + + @JsonIgnore + public long getCreatedTimeMillis() { + return 0L; + } + + @JsonIgnore + public long getUpdatedTimeMillis() { + return 0L; + } + +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceSearchResults.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceSearchResults.java new file mode 100644 index 0000000000..fc55ad891a --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceSearchResults.java @@ -0,0 +1,40 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + + +package org.opensearch.dataprepper.plugins.source.confluence.models; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Getter; + +import java.util.List; + +/** + * The result of a CQL search. + */ +@Getter +@JsonIgnoreProperties(ignoreUnknown = true) +public class ConfluenceSearchResults { + + + @JsonProperty("start") + private Integer start = null; + + @JsonProperty("limit") + private Integer limit = null; + + @JsonProperty("size") + private Integer size = null; + + @JsonProperty("results") + private List results = null; + +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/SpaceItem.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/SpaceItem.java new file mode 100644 index 0000000000..9e8733b582 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/models/SpaceItem.java @@ -0,0 +1,61 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.models; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Getter; +import lombok.Setter; + + +@Setter +@Getter +@JsonIgnoreProperties(ignoreUnknown = true) +public class SpaceItem { + + /** + * The ID of the issue. + */ + @JsonProperty("id") + private int id; + + /** + * The type of the issue. + */ + @JsonProperty("key") + private String key = null; + + /** + * The type of the issue. + */ + @JsonProperty("alias") + private String alias = null; + + /** + * The type of the issue. + */ + @JsonProperty("name") + private String name = null; + + /** + * The type of the issue. + */ + @JsonProperty("status") + private String status = null; + + /** + * The type of the issue. + */ + @JsonProperty("type") + private String type = null; + + +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClient.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClient.java new file mode 100644 index 0000000000..05367f20f9 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClient.java @@ -0,0 +1,100 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.rest; + +import io.micrometer.core.annotation.Timed; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.Timer; +import lombok.extern.slf4j.Slf4j; +import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.plugins.source.atlassian.rest.AtlassianRestClient; +import org.opensearch.dataprepper.plugins.source.atlassian.rest.auth.AtlassianAuthConfig; +import org.opensearch.dataprepper.plugins.source.confluence.models.ConfluenceSearchResults; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.util.UriComponentsBuilder; + +import javax.inject.Named; +import java.net.URI; + +import static org.opensearch.dataprepper.plugins.source.confluence.utils.CqlConstants.CQL_FIELD; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.CqlConstants.EXPAND_FIELD; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.CqlConstants.EXPAND_VALUE; + +@Slf4j +@Named +public class ConfluenceRestClient extends AtlassianRestClient { + + public static final String REST_API_SEARCH = "wiki/rest/api/content/search"; + public static final String REST_API_FETCH_CONTENT = "wiki/rest/api/content/"; + public static final String REST_API_CONTENT_EXPAND_PARAM = "?expand=body.view"; + //public static final String REST_API_SPACES = "/rest/api/api/spaces"; + public static final String FIFTY = "50"; + public static final String START_AT = "startAt"; + public static final String MAX_RESULT = "maxResults"; + private static final String PAGE_FETCH_LATENCY_TIMER = "pageFetchLatency"; + private static final String SEARCH_CALL_LATENCY_TIMER = "searchCallLatency"; + private static final String SPACES_FETCH_LATENCY_TIMER = "spacesFetchLatency"; + private static final String PAGES_REQUESTED = "pagesRequested"; + private final RestTemplate restTemplate; + private final AtlassianAuthConfig authConfig; + private final Timer contentFetchLatencyTimer; + private final Timer searchCallLatencyTimer; + private final Timer spaceFetchLatencyTimer; + private final Counter contentRequestedCounter; + + public ConfluenceRestClient(RestTemplate restTemplate, AtlassianAuthConfig authConfig, + PluginMetrics pluginMetrics) { + super(restTemplate, authConfig); + this.restTemplate = restTemplate; + this.authConfig = authConfig; + + contentFetchLatencyTimer = pluginMetrics.timer(PAGE_FETCH_LATENCY_TIMER); + searchCallLatencyTimer = pluginMetrics.timer(SEARCH_CALL_LATENCY_TIMER); + spaceFetchLatencyTimer = pluginMetrics.timer(SPACES_FETCH_LATENCY_TIMER); + contentRequestedCounter = pluginMetrics.counter(PAGES_REQUESTED); + } + + /** + * Method to get all Contents in a paginated fashion. + * + * @param cql input parameter. + * @param startAt the start at + * @return InputStream input stream + */ + @Timed(SEARCH_CALL_LATENCY_TIMER) + public ConfluenceSearchResults getAllContent(StringBuilder cql, int startAt) { + + String url = authConfig.getUrl() + REST_API_SEARCH; + + URI uri = UriComponentsBuilder.fromHttpUrl(url) + .queryParam(MAX_RESULT, FIFTY) + .queryParam(START_AT, startAt) + .queryParam(CQL_FIELD, cql) + .queryParam(EXPAND_FIELD, EXPAND_VALUE) + .buildAndExpand().toUri(); + return invokeRestApi(uri, ConfluenceSearchResults.class).getBody(); + } + + /** + * Fetches content based on given the content id. + * + * @param contentId the item info + * @return the content based on the given content id + */ + @Timed(PAGE_FETCH_LATENCY_TIMER) + public String getContent(String contentId) { + contentRequestedCounter.increment(); + String url = authConfig.getUrl() + REST_API_FETCH_CONTENT + "/" + contentId + REST_API_CONTENT_EXPAND_PARAM; + URI uri = UriComponentsBuilder.fromHttpUrl(url).buildAndExpand().toUri(); + return invokeRestApi(uri, String.class).getBody(); + } + +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/ConfluenceConfigHelper.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/ConfluenceConfigHelper.java new file mode 100644 index 0000000000..39eecc8417 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/ConfluenceConfigHelper.java @@ -0,0 +1,102 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.utils; + + +import lombok.extern.slf4j.Slf4j; +import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceSourceConfig; +import org.opensearch.dataprepper.plugins.source.source_crawler.utils.AddressValidation; + +import java.util.ArrayList; +import java.util.List; + +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.BASIC; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.OAUTH2; + +/** + * The type Confluence configuration. + */ +@Slf4j +public class ConfluenceConfigHelper { + + /** + * Get Content Types Filter from configuration. + * + * @return List Content Type Filter. + */ + public static List getContentTypeIncludeFilter(ConfluenceSourceConfig repositoryConfiguration) { + if (repositoryConfiguration.getFilterConfig() == null || repositoryConfiguration.getFilterConfig().getPageTypeConfig() == null) { + return new ArrayList<>(); + } + return repositoryConfiguration.getFilterConfig().getPageTypeConfig().getInclude(); + } + + public static List getContentTypeExcludeFilter(ConfluenceSourceConfig repositoryConfiguration) { + if (repositoryConfiguration.getFilterConfig() == null || repositoryConfiguration.getFilterConfig().getPageTypeConfig() == null) { + return new ArrayList<>(); + } + return repositoryConfiguration.getFilterConfig().getPageTypeConfig().getExclude(); + } + + /** + * Get Space Filter Types from configuration. + * + * @return List Space Filter. + */ + public static List getSpacesNameIncludeFilter(ConfluenceSourceConfig repositoryConfiguration) { + if (repositoryConfiguration.getFilterConfig() == null || + repositoryConfiguration.getFilterConfig().getSpaceConfig() == null || + repositoryConfiguration.getFilterConfig().getSpaceConfig().getNameConfig() == null) { + return new ArrayList<>(); + } + return repositoryConfiguration.getFilterConfig().getSpaceConfig().getNameConfig().getInclude(); + } + + public static List getSpacesNameExcludeFilter(ConfluenceSourceConfig repositoryConfiguration) { + if (repositoryConfiguration.getFilterConfig() == null || + repositoryConfiguration.getFilterConfig().getSpaceConfig() == null || + repositoryConfiguration.getFilterConfig().getSpaceConfig().getNameConfig() == null) { + return new ArrayList<>(); + } + return repositoryConfiguration.getFilterConfig().getSpaceConfig().getNameConfig().getExclude(); + } + + + public static boolean validateConfig(ConfluenceSourceConfig config) { + if (config.getAccountUrl() == null) { + throw new RuntimeException("Account URL is missing."); + } + //At least one of the AuthType should be present + if (config.getAuthType() == null) { + throw new RuntimeException("Authentication Type is missing."); + } + String authType = config.getAuthType(); + if (!OAUTH2.equals(authType) && !BASIC.equals(authType)) { + throw new RuntimeException("Invalid AuthType is given"); + } + + if (BASIC.equals(authType)) { + if (config.getAuthenticationConfig().getBasicConfig().getUsername() == null || config.getAuthenticationConfig().getBasicConfig().getPassword() == null) { + throw new RuntimeException("Confluence ID or Credential are required for Basic AuthType"); + } + } + + if (OAUTH2.equals(authType)) { + if (config.getAuthenticationConfig().getOauth2Config().getAccessToken() == null || config.getAuthenticationConfig().getOauth2Config().getRefreshToken() == null) { + throw new RuntimeException("Access Token or Refresh Token are required for OAuth2 AuthType"); + } + } + + AddressValidation.validateInetAddress(AddressValidation + .getInetAddress(config.getAccountUrl())); + return true; + } +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/ConfluenceContentType.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/ConfluenceContentType.java new file mode 100644 index 0000000000..20485b1552 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/ConfluenceContentType.java @@ -0,0 +1,35 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.utils; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +@AllArgsConstructor +public enum ConfluenceContentType { + SPACE("SPACE"), + PAGE("PAGE"), + BLOGPOST("BLOGPOST"), + COMMENT("COMMENT"), + ATTACHMENT("ATTACHMENT"); + + @Getter + private final String type; + + public static ConfluenceContentType fromString(String value) { + for (ConfluenceContentType contentType : ConfluenceContentType.values()) { + if (contentType.type.equalsIgnoreCase(value)) { + return contentType; + } + } + return null; + } +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/Constants.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/Constants.java new file mode 100644 index 0000000000..bb1460c51e --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/Constants.java @@ -0,0 +1,27 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.utils; + +/** + * The type Constants. + */ +public class Constants { + + public static final String SPACE = "space"; + public static final String OAUTH2 = "OAuth2"; + public static final String LAST_MODIFIED = "lastModified"; + public static final String SPACE_KEY = "space"; + public static final String CONTENT_TITLE = "title"; + public static final String CONTENT_ID = "id"; + public static final String CREATED = "created"; + public static final String BASIC = "Basic"; + public static final String PLUGIN_NAME = "confluence"; +} \ No newline at end of file diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/CqlConstants.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/CqlConstants.java new file mode 100644 index 0000000000..9d0de95966 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/CqlConstants.java @@ -0,0 +1,27 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.utils; + +public class CqlConstants { + public static final String GREATER_THAN_EQUALS = ">="; + public static final String CLOSING_ROUND_BRACKET = ")"; + + public static final String SPACE_IN = " AND space in ("; + public static final String SPACE_NOT_IN = " AND space not in ("; + public static final String DELIMITER = "\",\""; + public static final String PREFIX = "\""; + public static final String SUFFIX = "\""; + public static final String CONTENT_TYPE_IN = " AND type in ("; + public static final String CONTENT_TYPE_NOT_IN = " AND type not in ("; + public static final String CQL_FIELD = "cql"; + public static final String EXPAND_FIELD = "expand"; + public static final String EXPAND_VALUE = "all,space"; +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/HtmlToTextConversionUtil.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/HtmlToTextConversionUtil.java new file mode 100644 index 0000000000..3d34dbf8e3 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/main/java/org/opensearch/dataprepper/plugins/source/confluence/utils/HtmlToTextConversionUtil.java @@ -0,0 +1,53 @@ +package org.opensearch.dataprepper.plugins.source.confluence.utils; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; + +import java.util.Optional; + +public class HtmlToTextConversionUtil { + + public static JsonNode convertHtmlToText(ObjectNode jsonObject, String path) { + Optional valueAtGivenPath = getValueAtGivenPath(jsonObject, path); + if (valueAtGivenPath.isPresent()) { + String html = valueAtGivenPath.get().textValue(); + String txtBody = convertHtmlToText(html); + setValueAtGivenPath(jsonObject, path, txtBody); + } + return jsonObject; + } + + public static void setValueAtGivenPath(ObjectNode jsonObject, String path, String value) { + String[] keys = path.split("/"); + JsonNode current = jsonObject; + for (int i = 0; i < keys.length - 1; i++) { + current = current.get(keys[i]); + } + ((ObjectNode) current).put(keys[keys.length - 1], value); + } + + public static Optional getValueAtGivenPath(ObjectNode jsonObject, String path) { + try { + String[] keys = path.split("/"); + ObjectNode current = jsonObject; + for (int i = 0; i < keys.length - 1; i++) { + current = (ObjectNode) current.get(keys[i]); + } + return Optional.of(current.get(keys[keys.length - 1])); + } catch (Exception e) { + return Optional.empty(); + } + } + + public static String convertHtmlToText(String html) { + if (html == null || html.isEmpty()) { + return ""; + } + Document document = Jsoup.parse(html); + // Remove scripts and style elements + document.select("script, style").remove(); + return document.text(); + } +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceClientTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceClientTest.java new file mode 100644 index 0000000000..efd8e20303 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceClientTest.java @@ -0,0 +1,145 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSet; +import org.opensearch.dataprepper.model.buffer.Buffer; +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.record.Record; +import org.opensearch.dataprepper.plugins.source.source_crawler.base.PluginExecutorServiceProvider; +import org.opensearch.dataprepper.plugins.source.source_crawler.coordination.state.SaasWorkerProgressState; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +public class ConfluenceClientTest { + + @Mock + private Buffer> buffer; + @Mock + private SaasWorkerProgressState saasWorkerProgressState; + @Mock + private AcknowledgementSet acknowledgementSet; + @Mock + private ConfluenceSourceConfig confluenceSourceConfig; + @Mock + private ConfluenceService confluenceService; + @Mock + private ConfluenceIterator confluenceIterator; + private final PluginExecutorServiceProvider executorServiceProvider = new PluginExecutorServiceProvider(); + + @Test + void testConstructor() { + ConfluenceClient confluenceClient = new ConfluenceClient(confluenceService, confluenceIterator, executorServiceProvider, confluenceSourceConfig); + confluenceClient.setLastPollTime(Instant.ofEpochSecond(1234L)); + assertNotNull(confluenceClient); + } + + @Test + void testListItems() { + ConfluenceClient confluenceClient = new ConfluenceClient(confluenceService, confluenceIterator, executorServiceProvider, confluenceSourceConfig); + assertNotNull(confluenceClient.listItems()); + } + + + @Test + void testExecutePartition() throws Exception { + ConfluenceClient confluenceClient = new ConfluenceClient(confluenceService, confluenceIterator, executorServiceProvider, confluenceSourceConfig); + Map keyAttributes = new HashMap<>(); + keyAttributes.put("project", "test"); + when(saasWorkerProgressState.getKeyAttributes()).thenReturn(keyAttributes); + List itemIds = new ArrayList<>(); + itemIds.add(null); + itemIds.add("ID2"); + itemIds.add("ID3"); + when(saasWorkerProgressState.getItemIds()).thenReturn(itemIds); + Instant exportStartTime = Instant.now(); + when(saasWorkerProgressState.getExportStartTime()).thenReturn(Instant.ofEpochSecond(exportStartTime.toEpochMilli())); + + when(confluenceService.getContent(anyString())).thenReturn("{\"id\":\"ID1\",\"key\":\"TEST-1\"}"); + + ArgumentCaptor>> recordsCaptor = ArgumentCaptor.forClass((Class) Collection.class); + + confluenceClient.executePartition(saasWorkerProgressState, buffer, acknowledgementSet); + + verify(buffer).writeAll(recordsCaptor.capture(), anyInt()); + Collection> capturedRecords = recordsCaptor.getValue(); + assertFalse(capturedRecords.isEmpty()); + for (Record record : capturedRecords) { + assertNotNull(record.getData()); + } + } + + @Test + void testExecutePartitionError() throws Exception { + ConfluenceClient confluenceClient = new ConfluenceClient(confluenceService, confluenceIterator, executorServiceProvider, confluenceSourceConfig); + Map keyAttributes = new HashMap<>(); + keyAttributes.put("project", "test"); + when(saasWorkerProgressState.getKeyAttributes()).thenReturn(keyAttributes); + List itemIds = List.of("ID1", "ID2", "ID3", "ID4"); + when(saasWorkerProgressState.getItemIds()).thenReturn(itemIds); + Instant exportStartTime = Instant.now(); + when(saasWorkerProgressState.getExportStartTime()).thenReturn(Instant.ofEpochSecond(exportStartTime.toEpochMilli())); + + when(confluenceService.getContent(anyString())).thenReturn("{\"id\":\"ID1\",\"key\":\"TEST-1\"}"); + + + ObjectMapper mockObjectMapper = mock(ObjectMapper.class); + when(mockObjectMapper.readValue(any(String.class), any(TypeReference.class))).thenThrow(new JsonProcessingException("test") { + }); + confluenceClient.injectObjectMapper(mockObjectMapper); + + assertThrows(RuntimeException.class, () -> confluenceClient.executePartition(saasWorkerProgressState, buffer, acknowledgementSet)); + } + + @Test + void bufferWriteRuntimeTest() throws Exception { + ConfluenceClient confluenceClient = new ConfluenceClient(confluenceService, confluenceIterator, executorServiceProvider, confluenceSourceConfig); + Map keyAttributes = new HashMap<>(); + keyAttributes.put("project", "test"); + when(saasWorkerProgressState.getKeyAttributes()).thenReturn(keyAttributes); + List itemIds = List.of("ID1", "ID2", "ID3", "ID4"); + when(saasWorkerProgressState.getItemIds()).thenReturn(itemIds); + Instant exportStartTime = Instant.now(); + when(saasWorkerProgressState.getExportStartTime()).thenReturn(Instant.ofEpochSecond(exportStartTime.toEpochMilli())); + + when(confluenceService.getContent(anyString())).thenReturn("{\"id\":\"ID1\",\"key\":\"TEST-1\"}"); + + ArgumentCaptor>> recordsCaptor = ArgumentCaptor.forClass((Class) Collection.class); + + doThrow(new RuntimeException()).when(buffer).writeAll(recordsCaptor.capture(), anyInt()); + assertThrows(RuntimeException.class, () -> confluenceClient.executePartition(saasWorkerProgressState, buffer, acknowledgementSet)); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceConfigHelperTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceConfigHelperTest.java new file mode 100644 index 0000000000..80528646e4 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceConfigHelperTest.java @@ -0,0 +1,156 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.model.plugin.PluginConfigVariable; +import org.opensearch.dataprepper.plugins.source.atlassian.configuration.AuthenticationConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.configuration.BasicConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.configuration.Oauth2Config; +import org.opensearch.dataprepper.plugins.source.confluence.configuration.FilterConfig; +import org.opensearch.dataprepper.plugins.source.confluence.configuration.NameConfig; +import org.opensearch.dataprepper.plugins.source.confluence.configuration.PageTypeConfig; +import org.opensearch.dataprepper.plugins.source.confluence.configuration.SpaceConfig; +import org.opensearch.dataprepper.plugins.source.confluence.utils.ConfluenceConfigHelper; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.when; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.BASIC; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.OAUTH2; + +@ExtendWith(MockitoExtension.class) +public class ConfluenceConfigHelperTest { + + @Mock + ConfluenceSourceConfig confluenceSourceConfig; + + @Mock + FilterConfig filterConfig; + + @Mock + PageTypeConfig pageTypeConfig; + + @Mock + SpaceConfig spaceConfig; + + @Mock + NameConfig nameConfig; + + @Mock + AuthenticationConfig authenticationConfig; + + @Mock + BasicConfig basicConfig; + + @Mock + Oauth2Config oauth2Config; + + @Mock + PluginConfigVariable accessTokenPluginConfigVariable; + + @Mock + PluginConfigVariable refreshTokenPluginConfigVariable; + + @Test + void testInitialization() { + ConfluenceConfigHelper confluenceConfigHelper = new ConfluenceConfigHelper(); + assertNotNull(confluenceConfigHelper); + } + + + @Test + void testGetIssueTypeFilter() { + when(confluenceSourceConfig.getFilterConfig()).thenReturn(filterConfig); + when(filterConfig.getPageTypeConfig()).thenReturn(pageTypeConfig); + assertTrue(ConfluenceConfigHelper.getContentTypeIncludeFilter(confluenceSourceConfig).isEmpty()); + assertTrue(ConfluenceConfigHelper.getContentTypeExcludeFilter(confluenceSourceConfig).isEmpty()); + List issueTypeFilter = List.of("Bug", "Story"); + List issueTypeExcludeFilter = List.of("Bug2", "Story2"); + when(pageTypeConfig.getInclude()).thenReturn(issueTypeFilter); + when(pageTypeConfig.getExclude()).thenReturn(issueTypeExcludeFilter); + assertEquals(issueTypeFilter, ConfluenceConfigHelper.getContentTypeIncludeFilter(confluenceSourceConfig)); + assertEquals(issueTypeExcludeFilter, ConfluenceConfigHelper.getContentTypeExcludeFilter(confluenceSourceConfig)); + } + + @Test + void testGetProjectNameFilter() { + when(confluenceSourceConfig.getFilterConfig()).thenReturn(filterConfig); + when(filterConfig.getSpaceConfig()).thenReturn(spaceConfig); + when(spaceConfig.getNameConfig()).thenReturn(nameConfig); + assertTrue(ConfluenceConfigHelper.getSpacesNameIncludeFilter(confluenceSourceConfig).isEmpty()); + assertTrue(ConfluenceConfigHelper.getSpacesNameExcludeFilter(confluenceSourceConfig).isEmpty()); + List projectNameFilter = List.of("TEST", "TEST2"); + List projectNameExcludeFilter = List.of("TEST3", "TEST4"); + when(nameConfig.getInclude()).thenReturn(projectNameFilter); + when(nameConfig.getExclude()).thenReturn(projectNameExcludeFilter); + assertEquals(projectNameFilter, ConfluenceConfigHelper.getSpacesNameIncludeFilter(confluenceSourceConfig)); + assertEquals(projectNameExcludeFilter, ConfluenceConfigHelper.getSpacesNameExcludeFilter(confluenceSourceConfig)); + } + + + @Test + void testValidateConfig() { + assertThrows(RuntimeException.class, () -> ConfluenceConfigHelper.validateConfig(confluenceSourceConfig)); + + when(confluenceSourceConfig.getAccountUrl()).thenReturn("https://test.com"); + assertThrows(RuntimeException.class, () -> ConfluenceConfigHelper.validateConfig(confluenceSourceConfig)); + + when(confluenceSourceConfig.getAuthType()).thenReturn("fakeType"); + assertThrows(RuntimeException.class, () -> ConfluenceConfigHelper.validateConfig(confluenceSourceConfig)); + } + + @Test + void testValidateConfigBasic() { + when(confluenceSourceConfig.getAccountUrl()).thenReturn("https://test.com"); + when(confluenceSourceConfig.getAuthType()).thenReturn(BASIC); + when(confluenceSourceConfig.getAuthenticationConfig()).thenReturn(authenticationConfig); + when(authenticationConfig.getBasicConfig()).thenReturn(basicConfig); + assertThrows(RuntimeException.class, () -> ConfluenceConfigHelper.validateConfig(confluenceSourceConfig)); + + when(basicConfig.getUsername()).thenReturn("id"); + assertThrows(RuntimeException.class, () -> ConfluenceConfigHelper.validateConfig(confluenceSourceConfig)); + + when(basicConfig.getPassword()).thenReturn("credential"); + when(basicConfig.getUsername()).thenReturn(null); + assertThrows(RuntimeException.class, () -> ConfluenceConfigHelper.validateConfig(confluenceSourceConfig)); + + when(basicConfig.getUsername()).thenReturn("id"); + assertDoesNotThrow(() -> ConfluenceConfigHelper.validateConfig(confluenceSourceConfig)); + } + + @Test + void testValidateConfigOauth2() { + when(confluenceSourceConfig.getAccountUrl()).thenReturn("https://test.com"); + when(confluenceSourceConfig.getAuthType()).thenReturn(OAUTH2); + when(confluenceSourceConfig.getAuthenticationConfig()).thenReturn(authenticationConfig); + when(authenticationConfig.getOauth2Config()).thenReturn(oauth2Config); + assertThrows(RuntimeException.class, () -> ConfluenceConfigHelper.validateConfig(confluenceSourceConfig)); + + when(oauth2Config.getAccessToken()).thenReturn(accessTokenPluginConfigVariable); + assertThrows(RuntimeException.class, () -> ConfluenceConfigHelper.validateConfig(confluenceSourceConfig)); + + when(authenticationConfig.getOauth2Config().getRefreshToken()).thenReturn(refreshTokenPluginConfigVariable); + when(oauth2Config.getAccessToken()).thenReturn(null); + assertThrows(RuntimeException.class, () -> ConfluenceConfigHelper.validateConfig(confluenceSourceConfig)); + + when(oauth2Config.getAccessToken()).thenReturn(accessTokenPluginConfigVariable); + assertDoesNotThrow(() -> ConfluenceConfigHelper.validateConfig(confluenceSourceConfig)); + } +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceItemInfoTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceItemInfoTest.java new file mode 100644 index 0000000000..26ef017dec --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceItemInfoTest.java @@ -0,0 +1,108 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.plugins.source.confluence.utils.Constants; + +import java.time.Instant; +import java.util.Map; +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +public class ConfluenceItemInfoTest { + private String project; + private String issueType; + private String id; + private String itemId; + private Instant eventTime; + + @Mock + private Map metadata; + + @Mock + private Map newMetadata; + + @Mock + private ConfluenceItemInfo confluenceItemInfo; + + @BeforeEach + void setUP() { + issueType = "TestIssue"; + id = UUID.randomUUID().toString(); + project = "TestProject"; + itemId = UUID.randomUUID().toString(); + eventTime = Instant.ofEpochSecond(0); + confluenceItemInfo = new ConfluenceItemInfo(id, itemId, project, issueType, metadata, eventTime); + } + + @Test + void testGetters() { + assertEquals(confluenceItemInfo.getItemId(), itemId); + assertEquals(confluenceItemInfo.getId(), id); + assertEquals(confluenceItemInfo.getSpace(), project); + assertEquals(confluenceItemInfo.getContentType(), issueType); + assertEquals(confluenceItemInfo.getMetadata(), metadata); + assertEquals(confluenceItemInfo.getEventTime(), eventTime); + } + + @Test + void testGetKeyAttributes() { + assertInstanceOf(Map.class, confluenceItemInfo.getKeyAttributes()); + } + + @Test + void testSetter() { + confluenceItemInfo.setEventTime(Instant.now()); + assertNotEquals(confluenceItemInfo.getEventTime(), eventTime); + confluenceItemInfo.setItemId("newItemID"); + assertNotEquals(confluenceItemInfo.getItemId(), itemId); + confluenceItemInfo.setId("newID"); + assertNotEquals(confluenceItemInfo.getId(), id); + confluenceItemInfo.setSpace("newProject"); + assertNotEquals(confluenceItemInfo.getSpace(), project); + confluenceItemInfo.setMetadata(newMetadata); + assertNotEquals(confluenceItemInfo.getMetadata(), metadata); + confluenceItemInfo.setContentType("newIssueType"); + assertNotEquals(confluenceItemInfo.getContentType(), issueType); + + } + + @Test + void testGetPartitionKey() { + String partitionKey = confluenceItemInfo.getPartitionKey(); + assertTrue(partitionKey.contains(project)); + assertTrue(partitionKey.contains(issueType)); + } + + + @Test + void testGetLastModifiedAt() { + when(metadata.get(Constants.LAST_MODIFIED)).thenReturn("5"); + when(metadata.get(Constants.CREATED)).thenReturn("0"); + assertEquals(Instant.ofEpochMilli(5), confluenceItemInfo.getLastModifiedAt()); + + when(metadata.get(Constants.LAST_MODIFIED)).thenReturn("5"); + when(metadata.get(Constants.CREATED)).thenReturn("7"); + assertEquals(Instant.ofEpochMilli(7), confluenceItemInfo.getLastModifiedAt()); + } + +} \ No newline at end of file diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceIteratorTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceIteratorTest.java new file mode 100644 index 0000000000..3dfbe0be6e --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceIteratorTest.java @@ -0,0 +1,96 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.plugins.source.confluence.models.ConfluenceSearchResults; +import org.opensearch.dataprepper.plugins.source.confluence.rest.ConfluenceRestClient; +import org.opensearch.dataprepper.plugins.source.source_crawler.base.PluginExecutorServiceProvider; + +import java.time.Instant; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; + +@ExtendWith(MockitoExtension.class) +public class ConfluenceIteratorTest { + + @Mock + private ConfluenceSearchResults mockConfluenceSearchResults; + @Mock + private ConfluenceRestClient confluenceRestClient; + private ConfluenceService confluenceService; + @Mock + private ConfluenceSourceConfig confluenceSourceConfig; + private ConfluenceIterator confluenceIterator; + private final PluginMetrics pluginMetrics = PluginMetrics.fromNames("confluenceService", "aws"); + private final PluginExecutorServiceProvider executorServiceProvider = new PluginExecutorServiceProvider(); + + @BeforeEach + void setUp() { + confluenceService = spy(new ConfluenceService(confluenceSourceConfig, confluenceRestClient, pluginMetrics)); + } + + public ConfluenceIterator createObjectUnderTest() { + return new ConfluenceIterator(confluenceService, executorServiceProvider, confluenceSourceConfig); + } + + @Test + void testInitialization() { + confluenceIterator = createObjectUnderTest(); + assertNotNull(confluenceIterator); + confluenceIterator.initialize(Instant.ofEpochSecond(0)); + doReturn(mockConfluenceSearchResults).when(confluenceRestClient).getAllContent(any(StringBuilder.class), anyInt()); + assertFalse(confluenceIterator.hasNext()); + } + + @Test + void sleepInterruptionTest() { + confluenceIterator = createObjectUnderTest(); + confluenceIterator.initialize(Instant.ofEpochSecond(0)); + + Thread testThread = new Thread(() -> { + assertThrows(InterruptedException.class, () -> { + try { + confluenceIterator.hasNext(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + }); + + testThread.start(); + testThread.interrupt(); + } + + + @Test + void testStartCrawlerThreads() { + confluenceIterator = createObjectUnderTest(); + confluenceIterator.initialize(Instant.ofEpochSecond(0)); + confluenceIterator.hasNext(); + confluenceIterator.hasNext(); + assertEquals(1, confluenceIterator.showFutureList().size()); + } + + +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceServiceTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceServiceTest.java new file mode 100644 index 0000000000..37231948a7 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceServiceTest.java @@ -0,0 +1,273 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.model.plugin.PluginConfigVariable; +import org.opensearch.dataprepper.plugins.source.atlassian.configuration.Oauth2Config; +import org.opensearch.dataprepper.plugins.source.confluence.models.ConfluenceItem; +import org.opensearch.dataprepper.plugins.source.confluence.models.ConfluenceSearchResults; +import org.opensearch.dataprepper.plugins.source.confluence.models.SpaceItem; +import org.opensearch.dataprepper.plugins.source.confluence.rest.ConfluenceRestClient; +import org.opensearch.dataprepper.plugins.source.confluence.utils.MockPluginConfigVariableImpl; +import org.opensearch.dataprepper.plugins.source.source_crawler.base.PluginExecutorServiceProvider; +import org.opensearch.dataprepper.plugins.source.source_crawler.exception.BadRequestException; +import org.opensearch.dataprepper.plugins.source.source_crawler.model.ItemInfo; +import org.opensearch.dataprepper.test.helper.ReflectivelySetField; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.InputStream; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.ConcurrentLinkedQueue; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; +import static org.opensearch.dataprepper.plugins.source.atlassian.rest.auth.AtlassianOauthConfig.ACCESSIBLE_RESOURCES; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.BASIC; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.OAUTH2; + + +/** + * The type Jira service. + */ +@ExtendWith(MockitoExtension.class) +public class ConfluenceServiceTest { + + private static final Logger log = LoggerFactory.getLogger(ConfluenceServiceTest.class); + @Mock + private ConfluenceRestClient confluenceRestClient; + private final PluginExecutorServiceProvider executorServiceProvider = new PluginExecutorServiceProvider(); + private final PluginMetrics pluginMetrics = PluginMetrics.fromNames("confluenceService", "aws"); + + private static InputStream getResourceAsStream(String resourceName) { + InputStream inputStream = Thread.currentThread().getContextClassLoader().getResourceAsStream(resourceName); + if (inputStream == null) { + inputStream = ConfluenceServiceTest.class.getResourceAsStream("/" + resourceName); + } + return inputStream; + } + + public static ConfluenceSourceConfig createConfluenceConfigurationFromYaml(String fileName) { + ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory()); + try (InputStream inputStream = getResourceAsStream(fileName)) { + ConfluenceSourceConfig confluenceSourceConfig = objectMapper.readValue(inputStream, ConfluenceSourceConfig.class); + Oauth2Config oauth2Config = confluenceSourceConfig.getAuthenticationConfig().getOauth2Config(); + if (oauth2Config != null) { + ReflectivelySetField.setField(Oauth2Config.class, oauth2Config, "accessToken", + new MockPluginConfigVariableImpl("mockAccessToken")); + ReflectivelySetField.setField(Oauth2Config.class, oauth2Config, "refreshToken", + new MockPluginConfigVariableImpl("mockRefreshToken")); + } + return confluenceSourceConfig; + } catch (IOException ex) { + log.error("Failed to parse pipeline Yaml", ex); + } catch (Exception e) { + throw new RuntimeException(e); + } + return null; + } + + public static ConfluenceSourceConfig createConfluenceConfiguration(String auth_type, + List pageTypes, + List projectKey) throws JsonProcessingException { + PluginConfigVariable pcvAccessToken = null; + PluginConfigVariable pcvRefreshToken = null; + ObjectMapper objectMapper = new ObjectMapper(); + Map authenticationMap = new HashMap<>(); + Map basicMap = new HashMap<>(); + Map oauth2Map = new HashMap<>(); + if (auth_type.equals(BASIC)) { + basicMap.put("username", "test_username"); + basicMap.put("password", "test_password"); + authenticationMap.put("basic", basicMap); + } else if (auth_type.equals(OAUTH2)) { + oauth2Map.put("client_id", "test-client-id"); + oauth2Map.put("client_secret", "test-client-secret"); + pcvAccessToken = new MockPluginConfigVariableImpl("test-access-token"); + pcvRefreshToken = new MockPluginConfigVariableImpl("test-refresh-token"); + authenticationMap.put("oauth2", oauth2Map); + } + + Map jiraSourceConfigMap = new HashMap<>(); + List hosts = new ArrayList<>(); + hosts.add(ACCESSIBLE_RESOURCES); + + Map filterMap = new HashMap<>(); + Map spacesMap = new HashMap<>(); + Map contentTypeMap = new HashMap<>(); + + contentTypeMap.put("include", pageTypes); + filterMap.put("page_type", contentTypeMap); + + Map nameMap = new HashMap<>(); + nameMap.put("include", projectKey); + spacesMap.put("key", nameMap); + filterMap.put("space", spacesMap); + + + jiraSourceConfigMap.put("hosts", hosts); + jiraSourceConfigMap.put("authentication", authenticationMap); + jiraSourceConfigMap.put("filter", filterMap); + + String jiraSourceConfigJsonString = objectMapper.writeValueAsString(jiraSourceConfigMap); + ConfluenceSourceConfig confluenceSourceConfig = objectMapper.readValue(jiraSourceConfigJsonString, ConfluenceSourceConfig.class); + if (confluenceSourceConfig.getAuthenticationConfig().getOauth2Config() != null && pcvAccessToken != null) { + try { + ReflectivelySetField.setField(Oauth2Config.class, + confluenceSourceConfig.getAuthenticationConfig().getOauth2Config(), "accessToken", pcvAccessToken); + ReflectivelySetField.setField(Oauth2Config.class, + confluenceSourceConfig.getAuthenticationConfig().getOauth2Config(), "refreshToken", pcvRefreshToken); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + return confluenceSourceConfig; + } + + @AfterEach + void tearDown() { + executorServiceProvider.terminateExecutor(); + } + + @Test + void testJiraServiceInitialization() throws JsonProcessingException { + List contentType = new ArrayList<>(); + List spacesKey = new ArrayList<>(); + ConfluenceSourceConfig confluenceSourceConfig = createConfluenceConfiguration(BASIC, contentType, spacesKey); + ConfluenceService confluenceService = new ConfluenceService(confluenceSourceConfig, confluenceRestClient, pluginMetrics); + assertNotNull(confluenceService); + when(confluenceRestClient.getContent(anyString())).thenReturn("test String"); + assertNotNull(confluenceService.getContent("test Key")); + } + + @Test + public void testGetPages() throws JsonProcessingException { + List contentType = new ArrayList<>(); + List spaceKey = new ArrayList<>(); + contentType.add("PAGE"); + spaceKey.add("KAN"); + ConfluenceSourceConfig confluenceSourceConfig = createConfluenceConfiguration(BASIC, contentType, spaceKey); + ConfluenceService confluenceService = spy(new ConfluenceService(confluenceSourceConfig, confluenceRestClient, pluginMetrics)); + List mockPages = new ArrayList<>(); + ConfluenceItem item1 = createConfluenceItemBean(); + mockPages.add(item1); + ConfluenceItem item2 = createConfluenceItemBean(); + mockPages.add(item2); + ConfluenceItem item3 = createConfluenceItemBean(); + mockPages.add(item3); + + ConfluenceSearchResults mockConfluenceSearchResults = mock(ConfluenceSearchResults.class); + when(mockConfluenceSearchResults.getResults()).thenReturn(mockPages); + + doReturn(mockConfluenceSearchResults).when(confluenceRestClient).getAllContent(any(StringBuilder.class), anyInt()); + + Instant timestamp = Instant.ofEpochSecond(0); + Queue itemInfoQueue = new ConcurrentLinkedQueue<>(); + confluenceService.getPages(confluenceSourceConfig, timestamp, itemInfoQueue); + assertEquals(mockPages.size(), itemInfoQueue.size()); + } + + @Test + public void buildIssueItemInfoMultipleFutureThreads() throws JsonProcessingException { + List pageType = new ArrayList<>(); + List projectKey = new ArrayList<>(); + pageType.add("PAGE"); + ConfluenceSourceConfig confluenceSourceConfig = createConfluenceConfiguration(BASIC, pageType, projectKey); + ConfluenceService confluenceService = spy(new ConfluenceService(confluenceSourceConfig, confluenceRestClient, pluginMetrics)); + List mockIssues = new ArrayList<>(); + Random random = new Random(); + int numberOfIssues = random.nextInt(100); + for (int i = 0; i < numberOfIssues; i++) { + mockIssues.add(createConfluenceItemBean()); + } + + ConfluenceSearchResults mockConfluenceSearchResults = mock(ConfluenceSearchResults.class); + when(mockConfluenceSearchResults.getResults()).thenReturn(mockIssues); + + doReturn(mockConfluenceSearchResults).when(confluenceRestClient).getAllContent(any(StringBuilder.class), anyInt()); + + Instant timestamp = Instant.ofEpochSecond(0); + Queue itemInfoQueue = new ConcurrentLinkedQueue<>(); + confluenceService.getPages(confluenceSourceConfig, timestamp, itemInfoQueue); + assertEquals(numberOfIssues, itemInfoQueue.size()); + } + + @Test + public void testBadProjectKeys() throws JsonProcessingException { + List pageType = new ArrayList<>(); + List projectKey = new ArrayList<>(); + pageType.add("PAGE"); + projectKey.add("Bad Project Key"); + projectKey.add("A"); + projectKey.add("!@#$"); + projectKey.add("AAAAAAAAAAAAAA"); + + ConfluenceSourceConfig confluenceSourceConfig = createConfluenceConfiguration(BASIC, pageType, projectKey); + ConfluenceService confluenceService = new ConfluenceService(confluenceSourceConfig, confluenceRestClient, pluginMetrics); + + Instant timestamp = Instant.ofEpochSecond(0); + Queue itemInfoQueue = new ConcurrentLinkedQueue<>(); + + assertThrows(BadRequestException.class, () -> confluenceService.getPages(confluenceSourceConfig, timestamp, itemInfoQueue)); + } + + @Test + public void testGetPagesException() throws JsonProcessingException { + List pageType = new ArrayList<>(); + List projectKey = new ArrayList<>(); + pageType.add("Task"); + ConfluenceSourceConfig confluenceSourceConfig = createConfluenceConfiguration(BASIC, pageType, projectKey); + ConfluenceService confluenceService = spy(new ConfluenceService(confluenceSourceConfig, confluenceRestClient, pluginMetrics)); + + Instant timestamp = Instant.ofEpochSecond(0); + Queue itemInfoQueue = new ConcurrentLinkedQueue<>(); + + assertThrows(RuntimeException.class, () -> confluenceService.getPages(confluenceSourceConfig, timestamp, itemInfoQueue)); + } + + + private ConfluenceItem createConfluenceItemBean() { + ConfluenceItem confluenceItem = new ConfluenceItem(); + confluenceItem.setId(UUID.randomUUID().toString()); + confluenceItem.setTitle("issue_1_key"); + SpaceItem spaceItem = new SpaceItem(); + spaceItem.setId(new Random().nextInt()); + spaceItem.setKey(UUID.randomUUID().toString()); + confluenceItem.setSpaceItem(spaceItem); + return confluenceItem; + } + +} \ No newline at end of file diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceConfigTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceConfigTest.java new file mode 100644 index 0000000000..69535ae697 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceConfigTest.java @@ -0,0 +1,130 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; +import org.opensearch.dataprepper.model.plugin.PluginConfigVariable; +import org.opensearch.dataprepper.plugins.source.atlassian.configuration.Oauth2Config; +import org.opensearch.dataprepper.plugins.source.confluence.utils.MockPluginConfigVariableImpl; +import org.opensearch.dataprepper.test.helper.ReflectivelySetField; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.BASIC; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.OAUTH2; + +public class ConfluenceSourceConfigTest { + private ConfluenceSourceConfig confluenceSourceConfig; + private final PluginConfigVariable accessToken = new MockPluginConfigVariableImpl("access token test"); + private final PluginConfigVariable refreshToken = new MockPluginConfigVariableImpl("refresh token test"); + private final String clientId = "client id test"; + private final String clientSecret = "client secret test"; + private final String password = "test Jira Credential"; + private final String username = "test Jira Id"; + private final String accountUrl = "https://example.atlassian.net"; + private final List spacesList = new ArrayList<>(); + private final List contentTypeList = new ArrayList<>(); + + private ConfluenceSourceConfig createConfluenceSourceConfig(String authtype, boolean hasToken) throws Exception { + PluginConfigVariable pcvAccessToken = null; + PluginConfigVariable pcvRefreshToken = null; + Map configMap = new HashMap<>(); + List hosts = new ArrayList<>(); + hosts.add(accountUrl); + + configMap.put("hosts", hosts); + + Map authenticationMap = new HashMap<>(); + Map basicMap = new HashMap<>(); + Map oauth2Map = new HashMap<>(); + if (authtype.equals(BASIC)) { + basicMap.put("username", username); + basicMap.put("password", password); + authenticationMap.put("basic", basicMap); + } else if (authtype.equals(OAUTH2)) { + if (hasToken) { + pcvRefreshToken = refreshToken; + pcvAccessToken = accessToken; + } else { + oauth2Map.put("refresh_token", null); + } + oauth2Map.put("client_id", clientId); + oauth2Map.put("client_secret", clientSecret); + authenticationMap.put("oauth2", oauth2Map); + } + + configMap.put("authentication", authenticationMap); + + spacesList.add("space1"); + spacesList.add("space2"); + + contentTypeList.add("page"); + contentTypeList.add("blogpost"); + + Map filterMap = new HashMap<>(); + Map projectMap = new HashMap<>(); + Map issueTypeMap = new HashMap<>(); + Map statusMap = new HashMap<>(); + + issueTypeMap.put("include", contentTypeList); + filterMap.put("page_type", issueTypeMap); + + + Map nameMap = new HashMap<>(); + nameMap.put("include", spacesList); + projectMap.put("key", nameMap); + filterMap.put("space", projectMap); + + configMap.put("filter", filterMap); + + ObjectMapper objectMapper = new ObjectMapper(); + String jsonConfig = objectMapper.writeValueAsString(configMap); + ConfluenceSourceConfig config = objectMapper.readValue(jsonConfig, ConfluenceSourceConfig.class); + if (config.getAuthenticationConfig().getOauth2Config() != null && pcvAccessToken != null) { + ReflectivelySetField.setField(Oauth2Config.class, + config.getAuthenticationConfig().getOauth2Config(), "accessToken", pcvAccessToken); + ReflectivelySetField.setField(Oauth2Config.class, + config.getAuthenticationConfig().getOauth2Config(), "refreshToken", pcvRefreshToken); + } + return config; + } + + @Test + void testGetters() throws Exception { + confluenceSourceConfig = createConfluenceSourceConfig(BASIC, false); + assertEquals(confluenceSourceConfig.getFilterConfig().getPageTypeConfig().getInclude(), contentTypeList); + assertEquals(confluenceSourceConfig.getFilterConfig().getSpaceConfig().getNameConfig().getInclude(), spacesList); + assertEquals(confluenceSourceConfig.getAccountUrl(), accountUrl); + assertEquals(confluenceSourceConfig.getAuthenticationConfig().getBasicConfig().getPassword(), password); + assertEquals(confluenceSourceConfig.getAuthenticationConfig().getBasicConfig().getUsername(), username); + } + + @Test + void testFetchGivenOauthAttributeWrongAuthType() throws Exception { + confluenceSourceConfig = createConfluenceSourceConfig(BASIC, true); + assertThrows(RuntimeException.class, () -> confluenceSourceConfig.getAuthenticationConfig().getOauth2Config().getAccessToken()); + } + + @Test + void testFetchGivenOauthAtrribute() throws Exception { + confluenceSourceConfig = createConfluenceSourceConfig(OAUTH2, true); + assertEquals(accessToken, confluenceSourceConfig.getAuthenticationConfig().getOauth2Config().getAccessToken()); + assertEquals(refreshToken, confluenceSourceConfig.getAuthenticationConfig().getOauth2Config().getRefreshToken()); + assertEquals(clientId, confluenceSourceConfig.getAuthenticationConfig().getOauth2Config().getClientId()); + assertEquals(clientSecret, confluenceSourceConfig.getAuthenticationConfig().getOauth2Config().getClientSecret()); + } +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceTest.java new file mode 100644 index 0000000000..e6c62ee7fc --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/ConfluenceSourceTest.java @@ -0,0 +1,118 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSetManager; +import org.opensearch.dataprepper.model.buffer.Buffer; +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.plugin.PluginFactory; +import org.opensearch.dataprepper.model.record.Record; +import org.opensearch.dataprepper.model.source.coordinator.enhanced.EnhancedSourceCoordinator; +import org.opensearch.dataprepper.plugins.source.atlassian.configuration.AuthenticationConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.configuration.BasicConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.rest.auth.AtlassianAuthConfig; +import org.opensearch.dataprepper.plugins.source.source_crawler.base.Crawler; +import org.opensearch.dataprepper.plugins.source.source_crawler.base.PluginExecutorServiceProvider; + +import java.util.concurrent.ExecutorService; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.atLeast; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.opensearch.dataprepper.plugins.source.atlassian.rest.auth.AtlassianOauthConfig.ACCESSIBLE_RESOURCES; +import static org.opensearch.dataprepper.plugins.source.confluence.utils.Constants.BASIC; + +@ExtendWith(MockitoExtension.class) +public class ConfluenceSourceTest { + + @Mock + Buffer> buffer; + @Mock + AuthenticationConfig authenticationConfig; + @Mock + BasicConfig basicConfig; + @Mock + private PluginMetrics pluginMetrics; + @Mock + private ConfluenceSourceConfig confluenceSourceConfig; + @Mock + private AtlassianAuthConfig jiraOauthConfig; + @Mock + private PluginFactory pluginFactory; + @Mock + private AcknowledgementSetManager acknowledgementSetManager; + @Mock + private Crawler crawler; + @Mock + private EnhancedSourceCoordinator sourceCooridinator; + @Mock + private PluginExecutorServiceProvider executorServiceProvider; + @Mock + private ExecutorService executorService; + + @Test + void initialization() { + when(executorServiceProvider.get()).thenReturn(executorService); + ConfluenceSource source = new ConfluenceSource(pluginMetrics, confluenceSourceConfig, jiraOauthConfig, pluginFactory, acknowledgementSetManager, crawler, executorServiceProvider); + assertNotNull(source); + } + + @Test + void testStart() { + when(executorServiceProvider.get()).thenReturn(executorService); + ConfluenceSource source = new ConfluenceSource(pluginMetrics, confluenceSourceConfig, jiraOauthConfig, pluginFactory, acknowledgementSetManager, crawler, executorServiceProvider); + when(confluenceSourceConfig.getAccountUrl()).thenReturn(ACCESSIBLE_RESOURCES); + when(confluenceSourceConfig.getAuthType()).thenReturn(BASIC); + when(confluenceSourceConfig.getAuthenticationConfig()).thenReturn(authenticationConfig); + when(authenticationConfig.getBasicConfig()).thenReturn(basicConfig); + when(basicConfig.getUsername()).thenReturn("Test Id"); + when(basicConfig.getPassword()).thenReturn("Test Credential"); + + source.setEnhancedSourceCoordinator(sourceCooridinator); + source.start(buffer); + verify(executorService, atLeast(1)).submit(any(Runnable.class)); + } + + @Test + void testStop() { + when(executorServiceProvider.get()).thenReturn(executorService); + ConfluenceSource source = new ConfluenceSource(pluginMetrics, confluenceSourceConfig, jiraOauthConfig, pluginFactory, acknowledgementSetManager, crawler, executorServiceProvider); + when(confluenceSourceConfig.getAccountUrl()).thenReturn(ACCESSIBLE_RESOURCES); + when(confluenceSourceConfig.getAuthType()).thenReturn(BASIC); + when(confluenceSourceConfig.getAuthenticationConfig()).thenReturn(authenticationConfig); + when(authenticationConfig.getBasicConfig()).thenReturn(basicConfig); + when(basicConfig.getUsername()).thenReturn("Test Id"); + when(basicConfig.getPassword()).thenReturn("Test Credential"); + + source.setEnhancedSourceCoordinator(sourceCooridinator); + source.start(buffer); + source.stop(); + verify(executorService).shutdownNow(); + } + + @Test + void testStop_WhenNotStarted() { + when(executorServiceProvider.get()).thenReturn(executorService); + ConfluenceSource source = new ConfluenceSource(pluginMetrics, confluenceSourceConfig, jiraOauthConfig, pluginFactory, acknowledgementSetManager, crawler, executorServiceProvider); + + source.stop(); + + verify(executorService, never()).shutdown(); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceItemTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceItemTest.java new file mode 100644 index 0000000000..827d0824a0 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceItemTest.java @@ -0,0 +1,62 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.models; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +@ExtendWith(MockitoExtension.class) +public class ConfluenceItemTest { + + + private ConfluenceItem confluenceItemBean; + + @BeforeEach + void setup() { + confluenceItemBean = new ConfluenceItem(); + } + + @Test + public void testInitialization() { + assertNotNull(confluenceItemBean); + } + + @Test + public void testNull() { + assertNull(confluenceItemBean.getId()); + } + + @Test + void testNullCases() { + assertEquals(confluenceItemBean.getUpdatedTimeMillis(), 0); + } + + @Test + void testGivenDateField() { + assertEquals(confluenceItemBean.getCreatedTimeMillis(), 0L); + assertEquals(confluenceItemBean.getUpdatedTimeMillis(), 0L); + } + + @Test + public void testStringSettersAndGetters() { + String id = "idTest"; + + confluenceItemBean.setId(id); + assertEquals(confluenceItemBean.getId(), id); + } + +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceSearchResultsTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceSearchResultsTest.java new file mode 100644 index 0000000000..fe070c8b98 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/models/ConfluenceSearchResultsTest.java @@ -0,0 +1,41 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.models; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; + +import static org.junit.jupiter.api.Assertions.assertNotNull; + +@ExtendWith(MockitoExtension.class) +public class ConfluenceSearchResultsTest { + + private ConfluenceSearchResults confluenceSearchResults; + private final ObjectMapper objectMapper = new ObjectMapper(); + + @BeforeEach + public void setUp() throws JsonProcessingException { + String state = "{}"; + confluenceSearchResults = objectMapper.readValue(state, ConfluenceSearchResults.class); + } + + @Test + public void testConstructor() { + assertNotNull(confluenceSearchResults); + + } + + +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClientTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClientTest.java new file mode 100644 index 0000000000..9868b7cb5d --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/rest/ConfluenceRestClientTest.java @@ -0,0 +1,144 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.rest; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.plugins.source.atlassian.rest.auth.AtlassianAuthConfig; +import org.opensearch.dataprepper.plugins.source.atlassian.rest.auth.AtlassianAuthFactory; +import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceServiceTest; +import org.opensearch.dataprepper.plugins.source.confluence.ConfluenceSourceConfig; +import org.opensearch.dataprepper.plugins.source.confluence.models.ConfluenceSearchResults; +import org.opensearch.dataprepper.plugins.source.source_crawler.exception.BadRequestException; +import org.opensearch.dataprepper.plugins.source.source_crawler.exception.UnauthorizedException; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.RestTemplate; + +import java.net.URI; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +public class ConfluenceRestClientTest { + + @Mock + private StringBuilder jql; + + @Mock + private RestTemplate restTemplate; + + @Mock + private AtlassianAuthConfig authConfig; + + private final PluginMetrics pluginMetrics = PluginMetrics.fromNames("jiraRestClient", "aws"); + + private static Stream provideHttpStatusCodesWithExceptionClass() { + return Stream.of( + Arguments.of(HttpStatus.FORBIDDEN, UnauthorizedException.class), + Arguments.of(HttpStatus.UNAUTHORIZED, RuntimeException.class), + Arguments.of(HttpStatus.TOO_MANY_REQUESTS, RuntimeException.class), + Arguments.of(HttpStatus.INSUFFICIENT_STORAGE, RuntimeException.class) + ); + } + + @ParameterizedTest + @ValueSource(strings = {"basic-auth-confluence-pipeline.yaml"}) + public void testFetchingJiraIssue(String configFileName) { + String examplePageResponse = "{\"id\":\"123\",\"key\":\"key\",\"self\":\"https://example.com/rest/api/2/issue/123\"}"; + doReturn(new ResponseEntity<>(examplePageResponse, HttpStatus.OK)).when(restTemplate).getForEntity(any(URI.class), any(Class.class)); + ConfluenceSourceConfig confluenceSourceConfig = ConfluenceServiceTest.createConfluenceConfigurationFromYaml(configFileName); + AtlassianAuthConfig authConfig = new AtlassianAuthFactory(confluenceSourceConfig).getObject(); + ConfluenceRestClient confluenceRestClient = new ConfluenceRestClient(restTemplate, authConfig, pluginMetrics); + String pageDetails = confluenceRestClient.getContent("key"); + assertEquals(examplePageResponse, pageDetails); + } + + @ParameterizedTest + @MethodSource("provideHttpStatusCodesWithExceptionClass") + void testInvokeRestApiTokenExpired(HttpStatus statusCode, Class expectedExceptionType) { + ConfluenceRestClient confluenceRestClient = new ConfluenceRestClient(restTemplate, authConfig, pluginMetrics); + confluenceRestClient.setSleepTimeMultiplier(1); + when(authConfig.getUrl()).thenReturn("https://example.com/rest/api/2/issue/key"); + when(restTemplate.getForEntity(any(URI.class), any(Class.class))).thenThrow(new HttpClientErrorException(statusCode)); + assertThrows(expectedExceptionType, () -> confluenceRestClient.getContent("key")); + } + + @Test + void testInvokeRestApiTokenExpiredInterruptException() throws InterruptedException { + ConfluenceRestClient confluenceRestClient = new ConfluenceRestClient(restTemplate, authConfig, pluginMetrics); + when(authConfig.getUrl()).thenReturn("https://example.com/rest/api/2/issue/key"); + when(restTemplate.getForEntity(any(URI.class), any(Class.class))).thenThrow(new HttpClientErrorException(HttpStatus.TOO_MANY_REQUESTS)); + confluenceRestClient.setSleepTimeMultiplier(100000); + + Thread testThread = new Thread(() -> { + assertThrows(InterruptedException.class, () -> { + try { + confluenceRestClient.getContent("key"); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + }); + testThread.start(); + Thread.sleep(100); + testThread.interrupt(); + } + + @Test + public void testGetAllContentOauth2() { + List issueType = new ArrayList<>(); + issueType.add("Task"); + ConfluenceRestClient confluenceRestClient = new ConfluenceRestClient(restTemplate, authConfig, pluginMetrics); + ConfluenceSearchResults mockConfluenceSearchResults = mock(ConfluenceSearchResults.class); + doReturn("http://mock-service.jira.com/").when(authConfig).getUrl(); + doReturn(new ResponseEntity<>(mockConfluenceSearchResults, HttpStatus.OK)).when(restTemplate).getForEntity(any(URI.class), any(Class.class)); + ConfluenceSearchResults results = confluenceRestClient.getAllContent(jql, 0); + assertNotNull(results); + } + + @Test + public void testGetAllContentBasic() { + List issueType = new ArrayList<>(); + issueType.add("Task"); + ConfluenceRestClient confluenceRestClient = new ConfluenceRestClient(restTemplate, authConfig, pluginMetrics); + ConfluenceSearchResults mockConfluenceSearchResults = mock(ConfluenceSearchResults.class); + when(authConfig.getUrl()).thenReturn("https://example.com/"); + doReturn(new ResponseEntity<>(mockConfluenceSearchResults, HttpStatus.OK)).when(restTemplate).getForEntity(any(URI.class), any(Class.class)); + ConfluenceSearchResults results = confluenceRestClient.getAllContent(jql, 0); + assertNotNull(results); + } + + @Test + public void testRestApiAddressValidation() { + when(authConfig.getUrl()).thenReturn("https://224.0.0.1/"); + ConfluenceRestClient confluenceRestClient = new ConfluenceRestClient(restTemplate, authConfig, pluginMetrics); + assertThrows(BadRequestException.class, () -> confluenceRestClient.getContent("TEST-1")); + } + +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/utils/ConfluenceContentTypeTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/utils/ConfluenceContentTypeTest.java new file mode 100644 index 0000000000..b81bd22b13 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/utils/ConfluenceContentTypeTest.java @@ -0,0 +1,36 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.utils; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +public class ConfluenceContentTypeTest { + @Test + void testEnumConstants() { + assertNotNull(ConfluenceContentType.SPACE); + assertNotNull(ConfluenceContentType.PAGE); + assertNotNull(ConfluenceContentType.COMMENT); + assertNotNull(ConfluenceContentType.ATTACHMENT); + assertNotNull(ConfluenceContentType.BLOGPOST); + } + + @Test + void testTypeValues() { + assertEquals("SPACE", ConfluenceContentType.SPACE.getType()); + assertEquals("PAGE", ConfluenceContentType.PAGE.getType()); + assertEquals("COMMENT", ConfluenceContentType.COMMENT.getType()); + assertEquals("ATTACHMENT", ConfluenceContentType.ATTACHMENT.getType()); + assertEquals("BLOGPOST", ConfluenceContentType.BLOGPOST.getType()); + } +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/utils/HtmlToTextConversionUtilTest.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/utils/HtmlToTextConversionUtilTest.java new file mode 100644 index 0000000000..23d6421aea --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/utils/HtmlToTextConversionUtilTest.java @@ -0,0 +1,276 @@ +package org.opensearch.dataprepper.plugins.source.confluence.utils; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.Optional; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class HtmlToTextConversionUtilTest { + + private ObjectMapper objectMapper; + private ObjectNode jsonObject; + + private static Stream provideDifferentTypeValues() { + return Stream.of( + Arguments.of(42, "number"), + Arguments.of(true, "boolean"), + Arguments.of(3.14, "number"), + Arguments.of("string value", "string") + ); + } + + @BeforeEach + void setUp() { + objectMapper = new ObjectMapper(); + jsonObject = objectMapper.createObjectNode(); + } + + @Test + void convertHtmlToText_WithValidHtmlAndPath_ShouldConvertSuccessfully() { + // Arrange + jsonObject.put("content", "

Hello World

"); + String path = "content"; + + // Act + JsonNode result = HtmlToTextConversionUtil.convertHtmlToText(jsonObject, path); + + // Assert + assertTrue(result.has("content")); + assertEquals("Hello World", result.get("content").textValue()); + } + + @Test + void convertHtmlToText_WithNestedPath_ShouldConvertSuccessfully() { + // Arrange + ObjectNode nestedNode = objectMapper.createObjectNode(); + nestedNode.put("body", "

Hello World

"); + jsonObject.set("content", nestedNode); + String path = "content/body"; + + // Act + JsonNode result = HtmlToTextConversionUtil.convertHtmlToText(jsonObject, path); + + // Assert + assertTrue(result.has("content")); + assertTrue(result.get("content").has("body")); + assertEquals("Hello World", result.get("content").get("body").textValue()); + } + + @Test + void setValueAtGivenPath_WithSimplePath_ShouldSetValue() { + // Arrange + String path = "title"; + String value = "Test Title"; + + // Act + HtmlToTextConversionUtil.setValueAtGivenPath(jsonObject, path, value); + + // Assert + assertEquals(value, jsonObject.get("title").asText()); + } + + @Test + void setValueAtGivenPath_WithNestedPath_ShouldSetValue() { + // Arrange + jsonObject.putObject("content").putObject("body"); + String path = "content/body/text"; + String value = "Test Content"; + + // Act + HtmlToTextConversionUtil.setValueAtGivenPath(jsonObject, path, value); + + // Assert + assertEquals(value, jsonObject.get("content").get("body").get("text").asText()); + } + + @Test + void convertHtmlToText_WithNullHtml_ShouldReturnEmptyString() { + // Act + String result = HtmlToTextConversionUtil.convertHtmlToText(null); + + // Assert + assertEquals("", result); + } + + @Test + void convertHtmlToText_WithEmptyHtml_ShouldReturnEmptyString() { + // Act + String result = HtmlToTextConversionUtil.convertHtmlToText(""); + + // Assert + assertEquals("", result); + } + + @Test + void convertHtmlToText_WithComplexHtml_ShouldConvertToPlainText() { + // Arrange + String html = "

Title

This is a test paragraph

"; + + // Act + String result = HtmlToTextConversionUtil.convertHtmlToText(html); + + // Assert + assertEquals("Title This is a test paragraph", result.trim()); + } + + @Test + void convertHtmlToText_WithInvalidPath_ShouldReturnOriginalJson() { + // Arrange + jsonObject.put("content", "

Hello World

"); + String invalidPath = "invalid.path"; + + // Act + JsonNode result = HtmlToTextConversionUtil.convertHtmlToText(jsonObject, invalidPath); + + // Assert + assertEquals(jsonObject, result); + } + + @Test + void getValueAtGivenPath_WithSimplePath_ShouldReturnValue() { + // Arrange + String expectedValue = "test value"; + jsonObject.put("key", expectedValue); + + // Act + Optional result = HtmlToTextConversionUtil.getValueAtGivenPath(jsonObject, "key"); + + // Assert + assertTrue(result.isPresent()); + assertEquals(expectedValue, result.get().asText()); + } + + @Test + void getValueAtGivenPath_WithNestedPath_ShouldReturnValue() { + // Arrange + String expectedValue = "nested value"; + ObjectNode nestedNode = jsonObject.putObject("parent"); + nestedNode.put("child", expectedValue); + + // Act + Optional result = HtmlToTextConversionUtil.getValueAtGivenPath(jsonObject, "parent/child"); + + // Assert + assertTrue(result.isPresent()); + assertEquals(expectedValue, result.get().asText()); + } + + @Test + void getValueAtGivenPath_WithDeeplyNestedPath_ShouldReturnValue() { + // Arrange + String expectedValue = "deeply nested value"; + ObjectNode level1 = jsonObject.putObject("level1"); + ObjectNode level2 = level1.putObject("level2"); + level2.put("level3", expectedValue); + + // Act + Optional result = HtmlToTextConversionUtil.getValueAtGivenPath( + jsonObject, "level1/level2/level3"); + + // Assert + assertTrue(result.isPresent()); + assertEquals(expectedValue, result.get().asText()); + } + + @Test + void getValueAtGivenPath_WithNonExistentPath_ShouldReturnEmpty() { + // Act + Optional result = HtmlToTextConversionUtil.getValueAtGivenPath( + jsonObject, "nonexistent/path"); + + // Assert + assertFalse(result.isPresent()); + } + + @Test + void getValueAtGivenPath_WithNullPath_ShouldReturnEmpty() { + // Act + Optional result = HtmlToTextConversionUtil.getValueAtGivenPath(jsonObject, null); + + // Assert + assertFalse(result.isPresent()); + } + + @Test + void getValueAtGivenPath_WithEmptyPath_ShouldReturnEmpty() { + // Act + Optional result = HtmlToTextConversionUtil.getValueAtGivenPath(jsonObject, ""); + + // Assert + assertFalse(result.isPresent()); + } + + @ParameterizedTest + @MethodSource("provideDifferentTypeValues") + void getValueAtGivenPath_WithDifferentTypes_ShouldReturnCorrectValue(Object value, String expectedType) { + // Arrange + if (value instanceof Integer) { + jsonObject.put("key", (Integer) value); + } else if (value instanceof Boolean) { + jsonObject.put("key", (Boolean) value); + } else if (value instanceof Double) { + jsonObject.put("key", (Double) value); + } else { + jsonObject.put("key", String.valueOf(value)); + } + + // Act + Optional result = HtmlToTextConversionUtil.getValueAtGivenPath(jsonObject, "key"); + + // Assert + assertTrue(result.isPresent()); + assertEquals(value.toString(), result.get().asText()); + } + + @Test + void getValueAtGivenPath_WithInvalidIntermediatePath_ShouldReturnEmpty() { + // Arrange + jsonObject.put("key", "value"); + + // Act + Optional result = HtmlToTextConversionUtil.getValueAtGivenPath( + jsonObject, "nonexistent/key/child"); + + // Assert + assertFalse(result.isPresent()); + } + + @Test + void getValueAtGivenPath_WithNullIntermediateNode_ShouldReturnEmpty() { + // Arrange + ObjectNode parentNode = jsonObject.putObject("parent"); + parentNode.putNull("child"); + + // Act + Optional result = HtmlToTextConversionUtil.getValueAtGivenPath( + jsonObject, "parent/child/grandchild"); + + // Assert + assertFalse(result.isPresent()); + } + + @Test + void getValueAtGivenPath_WithArrayNode_ShouldReturnEmpty() { + // Arrange + jsonObject.putArray("array").add("value"); + + // Act + Optional result = HtmlToTextConversionUtil.getValueAtGivenPath( + jsonObject, "array/0"); + + // Assert + assertFalse(result.isPresent()); + } +} + diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/utils/MockPluginConfigVariableImpl.java b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/utils/MockPluginConfigVariableImpl.java new file mode 100644 index 0000000000..d474ac5002 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/java/org/opensearch/dataprepper/plugins/source/confluence/utils/MockPluginConfigVariableImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.confluence.utils; + +import org.opensearch.dataprepper.model.plugin.PluginConfigVariable; + +/** + * Mock implementation of PluginConfigVariable interface used only for Unit Testing. + */ +public class MockPluginConfigVariableImpl implements PluginConfigVariable { + + private Object defaultValue; + + public MockPluginConfigVariableImpl(Object defaultValue) { + this.defaultValue = defaultValue; + } + + @Override + public Object getValue() { + return null; + } + + @Override + public void setValue(Object someValue) { + this.defaultValue = someValue; + } + + @Override + public void refresh() { + } + + @Override + public boolean isUpdatable() { + return true; + } +} diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/resources/basic-auth-confluence-pipeline.yaml b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/resources/basic-auth-confluence-pipeline.yaml new file mode 100644 index 0000000000..0bfa6384e8 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/resources/basic-auth-confluence-pipeline.yaml @@ -0,0 +1,6 @@ +hosts: ["https://jira.com/"] +authentication: + basic: + username: "jiraId" + password: "jiraApiKey" + diff --git a/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/resources/oauth2-auth-confluence-pipeline.yaml b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/resources/oauth2-auth-confluence-pipeline.yaml new file mode 100644 index 0000000000..7a4afd3abf --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/confluence-source/src/test/resources/oauth2-auth-confluence-pipeline.yaml @@ -0,0 +1,6 @@ +hosts: [ "https://jira.com/" ] +authentication: + oauth2: + client_id: "client_id" + client_secret: "client_secret" + diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/exception/BadRequestException.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/exception/BadRequestException.java new file mode 100644 index 0000000000..4cbe181011 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/exception/BadRequestException.java @@ -0,0 +1,25 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.source_crawler.exception; + +/** + * Exception to indicate a bad REST call has been made. + * It could either be caused by bad user inputs or wrong url construction in the logic. + */ +public final class BadRequestException extends RuntimeException { + public BadRequestException(final String message, final Throwable throwable) { + super(message, throwable); + } + + public BadRequestException(final String message) { + super(message); + } +} diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/exception/UnauthorizedException.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/exception/UnauthorizedException.java new file mode 100644 index 0000000000..9b76b791bd --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/exception/UnauthorizedException.java @@ -0,0 +1,25 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.source_crawler.exception; + +/** + * Exception to indicate unauthorized access. + * It could either be caused by invalid credentials supplied by the user or failed renew the credentials. + */ +public final class UnauthorizedException extends RuntimeException { + public UnauthorizedException(final String message, final Throwable throwable) { + super(message, throwable); + } + + public UnauthorizedException(final String message) { + super(message); + } +} diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/AddressValidation.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/AddressValidation.java new file mode 100644 index 0000000000..b549fb2a5e --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/AddressValidation.java @@ -0,0 +1,60 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.source_crawler.utils; + +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +import org.opensearch.dataprepper.plugins.source.source_crawler.exception.BadRequestException; + +import java.net.InetAddress; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.UnknownHostException; + + +/** + * This is the AddressValidation Class. + */ + +@Slf4j +public class AddressValidation { + + public static final String INVALID_URL = "URL is not valid "; + + /** + * Method for getInetAddress. + * + * @param url input parameter. + */ + public static InetAddress getInetAddress(String url) { + try { + return InetAddress.getByName(new URL(url).getHost()); + } catch (UnknownHostException | MalformedURLException e) { + log.error(INVALID_URL, e); + throw new BadRequestException(e.getMessage(), e); + } + } + + /** + * Validates the InetAddress and throws if the address is any of the following: 1. Link Local + * Address 2. Loopback + * Address 3. Multicast Address 4. Any Local Address 5. Site Local Address + * + * @param address the {@link InetAddress} to validate. + * @throws BadRequestException if the address is invalid. + */ + public static void validateInetAddress(@NonNull final InetAddress address) { + if (address.isMulticastAddress() || address.isAnyLocalAddress() || address.isLinkLocalAddress() + || address.isSiteLocalAddress() || address.isLoopbackAddress()) { + throw new BadRequestException(INVALID_URL); + } + } +} \ No newline at end of file diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/exception/BadRequestExceptionTest.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/exception/BadRequestExceptionTest.java new file mode 100644 index 0000000000..249d0260b5 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/exception/BadRequestExceptionTest.java @@ -0,0 +1,64 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.source_crawler.exception; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.Mockito.mock; + +public class BadRequestExceptionTest { + private String message; + private Throwable throwable; + + @BeforeEach + void setUp() { + message = "Bad Request"; + throwable = mock(Throwable.class); + } + + @Nested + class MessageOnlyConstructor { + private BadRequestException createObjectUnderTest() { + return new BadRequestException(message); + } + + @Test + void getMessage_returns_message() { + assertEquals(createObjectUnderTest().getMessage(), message); + } + + @Test + void getCause_returns_null() { + assertNull(createObjectUnderTest().getCause()); + } + } + + @Nested + class MessageThrowableConstructor { + private BadRequestException createObjectUnderTest() { + return new BadRequestException(message, throwable); + } + + @Test + void getMessage_returns_message() { + assertEquals(createObjectUnderTest().getMessage(), message); + } + + @Test + void getCause_returns_throwable() { + assertEquals(createObjectUnderTest().getCause(), throwable); + } + } +} diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/exception/UnauthorizedExceptionTest.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/exception/UnauthorizedExceptionTest.java new file mode 100644 index 0000000000..d3fdb66826 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/exception/UnauthorizedExceptionTest.java @@ -0,0 +1,64 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.source_crawler.exception; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.Mockito.mock; + +public class UnauthorizedExceptionTest { + private String message; + private Throwable throwable; + + @BeforeEach + void setUp() { + message = "UnAuthorized Exception"; + throwable = mock(Throwable.class); + } + + @Nested + class MessageOnlyConstructor { + private UnauthorizedException createObjectUnderTest() { + return new UnauthorizedException(message); + } + + @Test + void getMessage_returns_message() { + assertEquals(createObjectUnderTest().getMessage(), message); + } + + @Test + void getCause_returns_null() { + assertNull(createObjectUnderTest().getCause()); + } + } + + @Nested + class MessageThrowableConstructor { + private UnauthorizedException createObjectUnderTest() { + return new UnauthorizedException(message, throwable); + } + + @Test + void getMessage_returns_message() { + assertEquals(createObjectUnderTest().getMessage(), message); + } + + @Test + void getCause_returns_throwable() { + assertEquals(createObjectUnderTest().getCause(), throwable); + } + } +} diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/AddressValidationTest.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/AddressValidationTest.java new file mode 100644 index 0000000000..c3e3e7a0e2 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/utils/AddressValidationTest.java @@ -0,0 +1,100 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.source_crawler.utils; + +import org.junit.jupiter.api.Test; +import org.opensearch.dataprepper.plugins.source.source_crawler.exception.BadRequestException; + +import java.net.InetAddress; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.UnknownHostException; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class AddressValidationTest { + + @Test + void testInstanceCreation() { + assertNotNull(new AddressValidation()); + } + + @Test + void testGetInetAddress() { + String testUrl = "https://www.amazon.com"; + System.out.print("Test output"); + System.out.print(AddressValidation.getInetAddress(testUrl)); + } + + @Test + void testGetInetAddressWithMalformedUrl() { + String testUrl = "XXXXXXXXXXXXXXXXXXXXXX"; + assertThrows(BadRequestException.class, () -> AddressValidation.getInetAddress(testUrl)); + } + + @Test + void testGetInetAddressWithUnknownHost() { + String testUrl = "https://www.thisurldoesntexist1384276t5917278481073.com"; + assertThrows(BadRequestException.class, () -> AddressValidation.getInetAddress(testUrl)); + } + + @Test + void testGetInetAddressWithNullUrl() { + String testUrl = null; + assertThrows(BadRequestException.class, () -> AddressValidation.getInetAddress(testUrl)); + } + + + @Test + void testValidateInetAddressAnyLocalAddress() throws UnknownHostException { + InetAddress wildcardAddress = InetAddress.getByName("0.0.0.0"); + assertThrows(BadRequestException.class, () -> AddressValidation.validateInetAddress(wildcardAddress)); + } + + @Test + void testValidateInetAddressMulticastAddress() throws UnknownHostException { + InetAddress multicastAddress = InetAddress.getByName("224.0.0.1"); + assertThrows(BadRequestException.class, () -> AddressValidation.validateInetAddress(multicastAddress)); + } + + @Test + void testValidateInetAddressLinkLocalAddress() throws UnknownHostException { + InetAddress linkLocalAddress = InetAddress.getByName("169.254.1.1"); + assertThrows(BadRequestException.class, () -> AddressValidation.validateInetAddress(linkLocalAddress)); + } + + @Test + void testValidateInetAddressSiteLocalAddress() throws UnknownHostException { + InetAddress siteLocalAddress = InetAddress.getByName("10.0.0.1"); + assertThrows(BadRequestException.class, () -> AddressValidation.validateInetAddress(siteLocalAddress)); + } + + @Test + void testValidateInetAddressLoopbackAddress() throws UnknownHostException { + InetAddress loopbackAddress = InetAddress.getByName("127.0.0.1"); + assertThrows(BadRequestException.class, () -> AddressValidation.validateInetAddress(loopbackAddress)); + } + + @Test + void testValidateInetAddressValidAddress() throws UnknownHostException, MalformedURLException { + InetAddress validAddress = InetAddress.getByName(new URL("https://www.amazon.com").getHost()); + assertDoesNotThrow(() -> AddressValidation.validateInetAddress(validAddress)); + } + + @Test + void testValidateInetAddressNullAddress() { + InetAddress nullAddress = null; + assertThrows(NullPointerException.class, () -> AddressValidation.validateInetAddress(nullAddress)); + } + +} diff --git a/settings.gradle b/settings.gradle index 37a125aaa5..55536cc7ab 100644 --- a/settings.gradle +++ b/settings.gradle @@ -49,6 +49,7 @@ dependencyResolutionManagement { version('spring', '5.3.39') library('spring-core', 'org.springframework', 'spring-core').versionRef('spring') library('spring-context', 'org.springframework', 'spring-context').versionRef('spring') + library('spring-web', 'org.springframework', 'spring-web').versionRef('spring') version('bouncycastle', '1.78.1') library('bouncycastle-bcprov', 'org.bouncycastle', 'bcprov-jdk18on').versionRef('bouncycastle') library('bouncycastle-bcpkix', 'org.bouncycastle', 'bcpkix-jdk18on').versionRef('bouncycastle') @@ -190,4 +191,6 @@ include 'data-prepper-plugins:opensearch-api-source' include 'data-prepper-plugins:saas-source-plugins' include 'data-prepper-plugins:saas-source-plugins:source-crawler' include 'data-prepper-plugins:saas-source-plugins:jira-source' +include 'data-prepper-plugins:saas-source-plugins:confluence-source' +include 'data-prepper-plugins:saas-source-plugins:atlassian-commons'