From 0cdbdad7e72373c191bf568d035fe254ae229dd9 Mon Sep 17 00:00:00 2001 From: Asif Sohail Mohammed Date: Tue, 13 Feb 2024 12:41:57 -0600 Subject: [PATCH] Addressed feedback Signed-off-by: Asif Sohail Mohammed --- .../geoip-processor/build.gradle | 6 +- .../plugins/processor/GeoIPProcessor.java | 79 ++++++++++-- .../processor/configuration/EntryConfig.java | 54 +------- ...DatabaseReaderInitializationException.java | 2 +- .../exception/DownloadFailedException.java | 2 +- .../exception/EngineFailureException.java | 12 ++ .../exception/InvalidIPAddressException.java | 12 ++ .../NoValidDatabaseFoundException.java | 2 +- .../extension/databasedownload/DBSource.java | 25 ---- .../databasedownload/DBSourceOptions.java | 2 +- .../GeoIPDatabaseManager.java | 5 +- .../databasedownload/GeoIPFileManager.java | 7 + .../HttpDBDownloadService.java | 5 +- .../LocalDBDownloadService.java | 4 +- ...vice.java => ManifestDownloadService.java} | 6 +- .../databasedownload/S3DBService.java | 3 +- .../utils/DatabaseSourceIdentification.java | 20 +-- .../processor/utils/IPValidationCheck.java | 13 +- .../plugins/processor/GeoIPProcessorTest.java | 120 +++++++++++++++--- .../configuration/EntryConfigTest.java | 69 ++-------- .../exception/EngineFailureExceptionTest.java | 33 +++++ .../InvalidIPAddressExceptionTest.java | 32 +++++ .../databasedownload/DBSourceTest.java | 45 ------- .../GeoIPDatabaseManagerTest.java | 8 +- .../GeoIPFileManagerTest.java | 36 ++++++ ....java => ManifestDownloadServiceTest.java} | 10 +- .../utils/DbSourceIdentificationTest.java | 2 +- .../utils/IPValidationCheckTest.java | 13 +- 28 files changed, 370 insertions(+), 257 deletions(-) create mode 100644 data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/exception/EngineFailureException.java create mode 100644 data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/exception/InvalidIPAddressException.java rename data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/{CDNDownloadService.java => ManifestDownloadService.java} (97%) create mode 100644 data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/exception/EngineFailureExceptionTest.java create mode 100644 data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/exception/InvalidIPAddressExceptionTest.java delete mode 100644 data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/DBSourceTest.java create mode 100644 data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/GeoIPFileManagerTest.java rename data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/{CDNDownloadServiceTest.java => ManifestDownloadServiceTest.java} (80%) diff --git a/data-prepper-plugins/geoip-processor/build.gradle b/data-prepper-plugins/geoip-processor/build.gradle index a5d3f37fd2..b9a85deefc 100644 --- a/data-prepper-plugins/geoip-processor/build.gradle +++ b/data-prepper-plugins/geoip-processor/build.gradle @@ -47,7 +47,7 @@ databaseNames.forEach { databaseName -> { def gradleName = databaseName.replaceAll('-', '') def downloadTask = tasks.register("download${gradleName}", Download) { src(url) - dest "build/resources/test/mmdb-files/geo-lite2/" + databaseName + ".mmdb" + dest "build/resources/test/mmdb-files/geo-lite2/${databaseName}.mmdb" overwrite true } downloadFiles.get().dependsOn downloadTask @@ -63,7 +63,7 @@ enterpriseDatabaseNames.forEach { enterpriseDatabaseName -> { def gradleName = enterpriseDatabaseName.replaceAll('-', '') def downloadEnterpriseTask = tasks.register("download${gradleName}", Download) { src(url) - dest "build/resources/test/mmdb-files/geo-ip2/" + enterpriseDatabaseName + ".mmdb" + dest "build/resources/test/mmdb-files/geo-ip2/${enterpriseDatabaseName}.mmdb" overwrite true } downloadFiles.get().dependsOn downloadEnterpriseTask @@ -84,7 +84,7 @@ jacocoTestCoverageVerification { violationRules { rule { limit { - minimum = 0.8 // temporarily reduce coverage for the builds to pass + minimum = 0.85 } } } diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessor.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessor.java index 0e5691fd1b..257a90d276 100644 --- a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessor.java +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessor.java @@ -17,6 +17,7 @@ import org.opensearch.dataprepper.plugins.processor.configuration.EntryConfig; import org.opensearch.dataprepper.plugins.processor.databaseenrich.GeoIPDatabaseReader; import org.opensearch.dataprepper.plugins.processor.exception.EnrichFailedException; +import org.opensearch.dataprepper.plugins.processor.exception.InvalidIPAddressException; import org.opensearch.dataprepper.plugins.processor.extension.GeoIPProcessorService; import org.opensearch.dataprepper.plugins.processor.extension.GeoIpConfigSupplier; import org.opensearch.dataprepper.plugins.processor.utils.IPValidationCheck; @@ -26,7 +27,10 @@ import java.net.InetAddress; import java.net.UnknownHostException; +import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -43,7 +47,6 @@ public class GeoIPProcessor extends AbstractProcessor, Record DATABASE_EXPIRED_TAGS = List.of("database_expired"); private final Counter geoIpEventsProcessed; private final Counter geoIpEventsFailedLookup; private final Counter geoIpEventsFailedEngineException; @@ -51,6 +54,8 @@ public class GeoIPProcessor extends AbstractProcessor, Record tagsOnFailure; private final GeoIPProcessorService geoIPProcessorService; private final ExpressionEvaluator expressionEvaluator; + private final Map> entryFieldsMap; + final Map> entryDatabaseMap; /** * GeoIPProcessor constructor for initialization of required attributes @@ -73,6 +78,9 @@ public GeoIPProcessor(final PluginMetrics pluginMetrics, this.geoIpEventsFailedLookup = pluginMetrics.counter(GEO_IP_EVENTS_FAILED_LOOKUP); //TODO: Use the exception metric for exceptions from service this.geoIpEventsFailedEngineException = pluginMetrics.counter(GEO_IP_EVENTS_FAILED_ENGINE_EXCEPTION); + + this.entryFieldsMap = populateGeoIPFields(); + this.entryDatabaseMap = populateGeoIPDatabases(); } /** @@ -99,7 +107,7 @@ public Collection> doExecute(final Collection> recor // TODO: Need to decide the behaviour, right now if all databases are expired we don't enrich the data. if (databasesExpired) { - event.getMetadata().addTags(DATABASE_EXPIRED_TAGS); + // TODO: Finalize the tags continue; } @@ -107,9 +115,17 @@ public Collection> doExecute(final Collection> recor for (final EntryConfig entry : geoIPProcessorConfig.getEntries()) { final String source = entry.getSource(); - final List fields = entry.getFields(); - final Set databases = entry.getGeoIPDatabases(); - final String ipAddress = event.get(source, String.class); + final List fields = entryFieldsMap.get(entry); + final Set databases = entryDatabaseMap.get(entry); + String ipAddress = null; + try { + ipAddress = event.get(source, String.class); + } catch (final Exception e) { + // add tags + LOG.error(DataPrepperMarkers.EVENT, "Failed to get IP address from [{}] in event: [{}]. Caused by:[{}]", + source, event, e.getMessage()); + } + //Lookup from DB if (ipAddress != null && !ipAddress.isEmpty()) { @@ -124,13 +140,13 @@ public Collection> doExecute(final Collection> recor } else { isEventFailedLookup = true; } - } catch (final EnrichFailedException ex) { - isEventFailedLookup = true; - LOG.error(DataPrepperMarkers.EVENT, "Failed to get Geo data for event: [{}] for the IP address [{}]. Caused by:{}", - event, ipAddress, ex.getMessage()); - } catch (final UnknownHostException e) { + } catch (final InvalidIPAddressException | UnknownHostException e) { isEventFailedLookup = true; LOG.error(DataPrepperMarkers.EVENT, "Failed to validate IP address: [{}] in event: [{}]. Caused by:[{}]", + ipAddress, event, e.getMessage()); + } catch (final EnrichFailedException e) { + isEventFailedLookup = true; + LOG.error(DataPrepperMarkers.EVENT, "Failed to get Geo data for event: [{}] for the IP address [{}]. Caused by:{}", event, ipAddress, e.getMessage()); } } else { @@ -148,6 +164,49 @@ public Collection> doExecute(final Collection> recor return records; } + private Map> populateGeoIPFields() { + final Map> entryConfigFieldsMap = new HashMap<>(); + for (final EntryConfig entry: geoIPProcessorConfig.getEntries()) { + final List includeFields = entry.getIncludeFields(); + final List excludeFields = entry.getExcludeFields(); + List geoIPFields = new ArrayList<>(); + if (includeFields != null && !includeFields.isEmpty()) { + for (final String field : includeFields) { + final GeoIPField geoIPField = GeoIPField.findByName(field); + if (geoIPField != null) { + geoIPFields.add(geoIPField); + } + } + } else if (excludeFields != null) { + final List excludeGeoIPFields = new ArrayList<>(); + for (final String field : excludeFields) { + final GeoIPField geoIPField = GeoIPField.findByName(field); + if (geoIPField != null) { + excludeGeoIPFields.add(geoIPField); + } + } + geoIPFields = new ArrayList<>(List.of(GeoIPField.values())); + geoIPFields.removeAll(excludeGeoIPFields); + } + entryConfigFieldsMap.put(entry, geoIPFields); + } + return entryConfigFieldsMap; + } + + private Map> populateGeoIPDatabases() { + final Map> entryConfigGeoIPDatabaseMap = new HashMap<>(); + for (final EntryConfig entry : geoIPProcessorConfig.getEntries()) { + final List geoIPFields = entryFieldsMap.get(entry); + final Set geoIPDatabasesToUse = new HashSet<>(); + for (final GeoIPField geoIPField : geoIPFields) { + final Set geoIPDatabases = geoIPField.getGeoIPDatabases(); + geoIPDatabasesToUse.addAll(geoIPDatabases); + } + entryConfigGeoIPDatabaseMap.put(entry, geoIPDatabasesToUse); + } + return entryConfigGeoIPDatabaseMap; + } + @Override public void prepareForShutdown() { } diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/configuration/EntryConfig.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/configuration/EntryConfig.java index c6b89f07b2..d9c22d6143 100644 --- a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/configuration/EntryConfig.java +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/configuration/EntryConfig.java @@ -5,17 +5,11 @@ package org.opensearch.dataprepper.plugins.processor.configuration; -import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import jakarta.validation.constraints.AssertTrue; import jakarta.validation.constraints.NotEmpty; -import org.opensearch.dataprepper.plugins.processor.GeoIPDatabase; -import org.opensearch.dataprepper.plugins.processor.GeoIPField; -import java.util.ArrayList; -import java.util.HashSet; import java.util.List; -import java.util.Set; public class EntryConfig { static final String DEFAULT_TARGET = "geo"; @@ -32,13 +26,6 @@ public class EntryConfig { @JsonProperty("exclude_fields") private List excludeFields; - @JsonIgnore - private List geoIPFields; - - @JsonIgnore - private Set geoIPDatabasesToUse; - - public String getSource() { return source; } @@ -47,45 +34,12 @@ public String getTarget() { return target; } - - public List getFields() { - if (geoIPFields != null) { - return geoIPFields; - } - geoIPFields = new ArrayList<>(); - if (includeFields != null) { - for(final String field: includeFields) { - final GeoIPField geoIPField = GeoIPField.findByName(field); - if (geoIPField != null) { - geoIPFields.add(geoIPField); - } - } - return geoIPFields; - } else if (excludeFields != null) { - final List excludeGeoIPFields = new ArrayList<>(); - for(final String field: excludeFields) { - final GeoIPField geoIPField = GeoIPField.findByName(field); - if (geoIPField != null) { - excludeGeoIPFields.add(geoIPField); - } - } - geoIPFields = new ArrayList<>(List.of(GeoIPField.values())); - geoIPFields.removeAll(excludeGeoIPFields); - return geoIPFields; - } - return geoIPFields; + public List getIncludeFields() { + return includeFields; } - public Set getGeoIPDatabases() { - if (geoIPDatabasesToUse != null) { - return geoIPDatabasesToUse; - } - geoIPDatabasesToUse = new HashSet<>(); - for (final GeoIPField geoIPField: getFields()) { - final Set geoIPDatabases = geoIPField.getGeoIPDatabases(); - geoIPDatabasesToUse.addAll(geoIPDatabases); - } - return geoIPDatabasesToUse; + public List getExcludeFields() { + return excludeFields; } @AssertTrue(message = "include_fields and exclude_fields are mutually exclusive. include_fields or exclude_fields is required.") diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/exception/DatabaseReaderInitializationException.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/exception/DatabaseReaderInitializationException.java index b0d58492f5..f51f249220 100644 --- a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/exception/DatabaseReaderInitializationException.java +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/exception/DatabaseReaderInitializationException.java @@ -5,7 +5,7 @@ package org.opensearch.dataprepper.plugins.processor.exception; -public class DatabaseReaderInitializationException extends RuntimeException { +public class DatabaseReaderInitializationException extends EngineFailureException { public DatabaseReaderInitializationException(final String exceptionMsg) { super(exceptionMsg); } diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/exception/DownloadFailedException.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/exception/DownloadFailedException.java index 43757731b7..05cf8d9613 100644 --- a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/exception/DownloadFailedException.java +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/exception/DownloadFailedException.java @@ -8,7 +8,7 @@ /** * Implementation class for DownloadFailedException Custom exception */ -public class DownloadFailedException extends RuntimeException { +public class DownloadFailedException extends EngineFailureException { public DownloadFailedException(final String exceptionMsg) { super(exceptionMsg); } diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/exception/EngineFailureException.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/exception/EngineFailureException.java new file mode 100644 index 0000000000..f5958b521a --- /dev/null +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/exception/EngineFailureException.java @@ -0,0 +1,12 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.processor.exception; + +public class EngineFailureException extends RuntimeException { + public EngineFailureException(final String exceptionMsg) { + super(exceptionMsg); + } +} diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/exception/InvalidIPAddressException.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/exception/InvalidIPAddressException.java new file mode 100644 index 0000000000..134941b5ad --- /dev/null +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/exception/InvalidIPAddressException.java @@ -0,0 +1,12 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.processor.exception; + +public class InvalidIPAddressException extends EnrichFailedException { + public InvalidIPAddressException(final String exceptionMsg) { + super(exceptionMsg); + } +} diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/exception/NoValidDatabaseFoundException.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/exception/NoValidDatabaseFoundException.java index ac191b80d0..218d8c0901 100644 --- a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/exception/NoValidDatabaseFoundException.java +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/exception/NoValidDatabaseFoundException.java @@ -5,7 +5,7 @@ package org.opensearch.dataprepper.plugins.processor.exception; -public class NoValidDatabaseFoundException extends RuntimeException { +public class NoValidDatabaseFoundException extends EngineFailureException { public NoValidDatabaseFoundException(final String exceptionMsg) { super(exceptionMsg); } diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/DBSource.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/DBSource.java index 41b343d8c9..8c30dd80a7 100644 --- a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/DBSource.java +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/DBSource.java @@ -5,17 +5,12 @@ package org.opensearch.dataprepper.plugins.processor.extension.databasedownload; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import javax.net.ssl.TrustManager; import javax.net.ssl.SSLContext; import javax.net.ssl.X509TrustManager; import javax.net.ssl.HttpsURLConnection; import javax.net.ssl.SSLSession; import javax.net.ssl.HostnameVerifier; -import java.io.File; -import java.io.UncheckedIOException; import java.security.KeyManagementException; import java.security.NoSuchAlgorithmException; import java.security.SecureRandom; @@ -24,28 +19,8 @@ import java.util.List; public interface DBSource { - - Logger LOG = LoggerFactory.getLogger(DBSource.class); void initiateDownload(List config) throws Exception; - /** - * create Folder If Not Exist - * @param outputFilePath Output File Path - * @return File - */ - static File createFolderIfNotExist(String outputFilePath) { - final File destFile = new File(outputFilePath); - try { - if (!destFile.exists()) { - destFile.mkdirs(); - } - } - catch (UncheckedIOException ex) { - LOG.info("Create Folder If NotExist Exception {0}", ex); - } - return destFile; - } - /** * initiateSSL * @throws NoSuchAlgorithmException NoSuchAlgorithmException diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/DBSourceOptions.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/DBSourceOptions.java index 1b776c40df..60eacff72a 100644 --- a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/DBSourceOptions.java +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/DBSourceOptions.java @@ -18,7 +18,7 @@ public enum DBSourceOptions { PATH("path"), URL("url"), S3("s3"), - CDN("cdn"); + HTTP_MANIFEST("http_manifest"); private final String option; diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/GeoIPDatabaseManager.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/GeoIPDatabaseManager.java index c5a7f9f8fb..4efd3cbdd2 100644 --- a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/GeoIPDatabaseManager.java +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/GeoIPDatabaseManager.java @@ -121,9 +121,10 @@ private void downloadDatabases() throws Exception { switchDirectory(); final String destinationPath = maxMindConfig.getDatabaseDestination() + File.separator + currentDatabaseDir; + geoIPFileManager.createDirectoryIfNotExist(destinationPath); switch (dbSourceOptions) { - case CDN: - dbSource = new CDNDownloadService(destinationPath); + case HTTP_MANIFEST: + dbSource = new ManifestDownloadService(destinationPath); dbSource.initiateDownload(databasePaths); downloadReady =true; break; diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/GeoIPFileManager.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/GeoIPFileManager.java index 9ac422d769..66dda2db70 100644 --- a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/GeoIPFileManager.java +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/GeoIPFileManager.java @@ -25,4 +25,11 @@ public void deleteFile(final File file) { file.delete(); } + public void createDirectoryIfNotExist(final String outputFilePath) { + final File destFile = new File(outputFilePath); + if (!destFile.exists()) { + destFile.mkdirs(); + } + } + } diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/HttpDBDownloadService.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/HttpDBDownloadService.java index f3be979c41..d28f6ab67e 100644 --- a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/HttpDBDownloadService.java +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/HttpDBDownloadService.java @@ -44,15 +44,14 @@ public HttpDBDownloadService(final String destinationDirectory, final GeoIPFileM * @param urlList urlList */ public void initiateDownload(List urlList) { - final File tmpDir = DBSource.createFolderIfNotExist(destinationDirectory); final String tarDir = destinationDirectory + File.separator + "tar"; final String downloadTarFilepath = tarDir + File.separator + "out.tar.gz"; for(final String url : urlList) { - DBSource.createFolderIfNotExist(tarDir); + geoIPFileManager.createDirectoryIfNotExist(tarDir); try { initiateSSL(); buildRequestAndDownloadFile(url, downloadTarFilepath); - decompressAndUntarFile(tarDir, downloadTarFilepath, tmpDir); + decompressAndUntarFile(tarDir, downloadTarFilepath, new File(destinationDirectory)); deleteTarFolder(tarDir); } catch (Exception ex) { LOG.info("InitiateDownload Exception {0} " , ex); diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/LocalDBDownloadService.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/LocalDBDownloadService.java index 085c016701..ae55c2343c 100644 --- a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/LocalDBDownloadService.java +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/LocalDBDownloadService.java @@ -31,10 +31,8 @@ public LocalDBDownloadService(final String destinationDirectory) { */ @Override public void initiateDownload(List config) throws Exception { - final String destPath = destinationDirectory; - DBSource.createFolderIfNotExist(destPath); File srcDatabaseConfigPath = new File(config.get(0)); - File destDatabaseConfigPath = new File(destPath); + File destDatabaseConfigPath = new File(destinationDirectory); FileUtils.copyDirectory(srcDatabaseConfigPath, destDatabaseConfigPath); } } diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/CDNDownloadService.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/ManifestDownloadService.java similarity index 97% rename from data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/CDNDownloadService.java rename to data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/ManifestDownloadService.java index b500fb88a8..37a9b86f17 100644 --- a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/CDNDownloadService.java +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/ManifestDownloadService.java @@ -22,7 +22,7 @@ import static org.opensearch.dataprepper.plugins.processor.databaseenrich.GeoIPDatabaseReader.MAXMIND_DATABASE_EXTENSION; -public class CDNDownloadService implements DBSource { +public class ManifestDownloadService implements DBSource { private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private static final int DEFAULT_BYTE_SIZE = 1024; private static final String CITY_DATABASE_NAME = "geolite2-city.mmdb"; @@ -35,14 +35,12 @@ public class CDNDownloadService implements DBSource { private Manifest countryManifest; private Manifest asnManifest; - public CDNDownloadService(final String directoryName) { + public ManifestDownloadService(final String directoryName) { this.directoryName = directoryName; } @Override public void initiateDownload(final List databasePaths) { - DBSource.createFolderIfNotExist(directoryName); - for(final String url : databasePaths) { final Manifest manifest = deserializeManifestFile(url); populateManifestObjects(manifest); diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/S3DBService.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/S3DBService.java index d5115fea9c..144facd034 100644 --- a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/S3DBService.java +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/S3DBService.java @@ -45,13 +45,12 @@ public S3DBService(final AwsAuthenticationOptionsConfig awsAuthenticationOptions * Initialisation of Download through Url * @param s3URLs s3URLs */ - public void initiateDownload(List s3URLs) { + public void initiateDownload(final List s3URLs) { for (String s3Url : s3URLs) { try { URI uri = new URI(s3Url); bucketName = uri.getHost(); bucketPath = removeTrailingSlash(removeLeadingSlash(uri.getPath())); - DBSource.createFolderIfNotExist(destinationDirectory); buildRequestAndDownloadFile(bucketName, bucketPath); } catch (URISyntaxException ex) { LOG.info("Initiate Download Exception", ex); diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/utils/DatabaseSourceIdentification.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/utils/DatabaseSourceIdentification.java index 079a72ded3..1afa8fda88 100644 --- a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/utils/DatabaseSourceIdentification.java +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/utils/DatabaseSourceIdentification.java @@ -25,7 +25,7 @@ private DatabaseSourceIdentification() { } private static final String S3_DOMAIN_PATTERN = "[a-zA-Z0-9-]+\\.s3\\.amazonaws\\.com"; - private static final String CDN_ENDPOINT_HOST = "devo.geoip.maps.opensearch.org"; + private static final String MANIFEST_ENDPOINT_PATH = "manifest.json"; /** * Check for database path is valid S3 URI or not @@ -53,7 +53,7 @@ public static boolean isURL(final String input) { try { final URI uri = new URI(input); final URL url = new URL(input); - return !uri.getHost().equals(CDN_ENDPOINT_HOST) && + return !input.endsWith(MANIFEST_ENDPOINT_PATH) && uri.getScheme() != null && !Pattern.matches(S3_DOMAIN_PATTERN, url.getHost()) && (uri.getScheme().equals("http") || uri.getScheme().equals("https")); @@ -78,15 +78,15 @@ public static boolean isFilePath(final String input) { * @return boolean */ public static boolean isCDNEndpoint(final String input) { - try { - final URI uri = new URI(input); - if (uri.getHost() != null) { - return uri.getHost().equals(CDN_ENDPOINT_HOST); + if (input.endsWith(MANIFEST_ENDPOINT_PATH)) { + try { + final URI uri = new URI(input); + return uri.getScheme().equals("http") || uri.getScheme().equals("https"); + } catch (final URISyntaxException e) { + return false; } - return false; - } catch (final URISyntaxException e) { - return false; } + return false; } /** @@ -102,7 +102,7 @@ public static DBSourceOptions getDatabasePathType(final List databasePat return DBSourceOptions.PATH; } else if (DatabaseSourceIdentification.isCDNEndpoint(databasePath)) { - downloadSourceOptions = DBSourceOptions.CDN; + downloadSourceOptions = DBSourceOptions.HTTP_MANIFEST; } else if(DatabaseSourceIdentification.isURL(databasePath)) { diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/utils/IPValidationCheck.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/utils/IPValidationCheck.java index d717ebc711..dceb1bc591 100644 --- a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/utils/IPValidationCheck.java +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/utils/IPValidationCheck.java @@ -5,6 +5,8 @@ package org.opensearch.dataprepper.plugins.processor.utils; +import org.opensearch.dataprepper.plugins.processor.exception.InvalidIPAddressException; + import java.net.Inet4Address; import java.net.Inet6Address; import java.net.InetAddress; @@ -20,10 +22,15 @@ public class IPValidationCheck { * Check for IP is valid or not * @param ipAddress ipAddress * @return boolean - * @throws UnknownHostException UnknownHostException + * @throws InvalidIPAddressException InvalidIPAddressException */ - public static boolean isPublicIpAddress(final String ipAddress) throws UnknownHostException { - InetAddress address = InetAddress.getByName(ipAddress); + public static boolean isPublicIpAddress(final String ipAddress) { + InetAddress address; + try { + address = InetAddress.getByName(ipAddress); + } catch (final UnknownHostException e) { + throw new InvalidIPAddressException(e.getMessage()); + } if (address instanceof Inet6Address || address instanceof Inet4Address) { return !address.isSiteLocalAddress() && !address.isLoopbackAddress(); } diff --git a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessorTest.java b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessorTest.java index 31a9fabc8e..573abff523 100644 --- a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessorTest.java +++ b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/GeoIPProcessorTest.java @@ -10,6 +10,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; import org.mockito.Mock; import org.mockito.MockedStatic; import org.mockito.junit.jupiter.MockitoExtension; @@ -37,6 +39,7 @@ import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; @@ -47,6 +50,30 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; +import static org.opensearch.dataprepper.plugins.processor.GeoIPField.CITY_CONFIDENCE; +import static org.opensearch.dataprepper.plugins.processor.GeoIPField.CONTINENT_CODE; +import static org.opensearch.dataprepper.plugins.processor.GeoIPField.CONTINENT_NAME; +import static org.opensearch.dataprepper.plugins.processor.GeoIPField.COUNTRY_CONFIDENCE; +import static org.opensearch.dataprepper.plugins.processor.GeoIPField.IS_COUNTRY_IN_EUROPEAN_UNION; +import static org.opensearch.dataprepper.plugins.processor.GeoIPField.LATITUDE; +import static org.opensearch.dataprepper.plugins.processor.GeoIPField.LEAST_SPECIFIED_SUBDIVISION_CONFIDENCE; +import static org.opensearch.dataprepper.plugins.processor.GeoIPField.LEAST_SPECIFIED_SUBDIVISION_ISO_CODE; +import static org.opensearch.dataprepper.plugins.processor.GeoIPField.LEAST_SPECIFIED_SUBDIVISION_NAME; +import static org.opensearch.dataprepper.plugins.processor.GeoIPField.LOCATION; +import static org.opensearch.dataprepper.plugins.processor.GeoIPField.LOCATION_ACCURACY_RADIUS; +import static org.opensearch.dataprepper.plugins.processor.GeoIPField.LONGITUDE; +import static org.opensearch.dataprepper.plugins.processor.GeoIPField.METRO_CODE; +import static org.opensearch.dataprepper.plugins.processor.GeoIPField.MOST_SPECIFIED_SUBDIVISION_CONFIDENCE; +import static org.opensearch.dataprepper.plugins.processor.GeoIPField.MOST_SPECIFIED_SUBDIVISION_ISO_CODE; +import static org.opensearch.dataprepper.plugins.processor.GeoIPField.MOST_SPECIFIED_SUBDIVISION_NAME; +import static org.opensearch.dataprepper.plugins.processor.GeoIPField.POSTAL_CODE; +import static org.opensearch.dataprepper.plugins.processor.GeoIPField.POSTAL_CODE_CONFIDENCE; +import static org.opensearch.dataprepper.plugins.processor.GeoIPField.REGISTERED_COUNTRY_ISO_CODE; +import static org.opensearch.dataprepper.plugins.processor.GeoIPField.REGISTERED_COUNTRY_NAME; +import static org.opensearch.dataprepper.plugins.processor.GeoIPField.REPRESENTED_COUNTRY_ISO_CODE; +import static org.opensearch.dataprepper.plugins.processor.GeoIPField.REPRESENTED_COUNTRY_NAME; +import static org.opensearch.dataprepper.plugins.processor.GeoIPField.REPRESENTED_COUNTRY_TYPE; +import static org.opensearch.dataprepper.plugins.processor.GeoIPField.TIME_ZONE; import static org.opensearch.dataprepper.plugins.processor.GeoIPProcessor.GEO_IP_EVENTS_FAILED_LOOKUP; import static org.opensearch.dataprepper.plugins.processor.GeoIPProcessor.GEO_IP_EVENTS_PROCESSED; @@ -72,6 +99,8 @@ class GeoIPProcessorTest { private Counter geoIpEventsFailedLookup; @Mock private GeoIPDatabaseReader geoIPDatabaseReader; + @Captor + private ArgumentCaptor> geoIPFieldCaptor; @BeforeEach void setUp() { @@ -92,14 +121,14 @@ private GeoIPProcessor createObjectUnderTest() { } @Test - void doExecuteTest_with_when_condition_should_only_enrich_events_that_match_when_condition() throws NoSuchFieldException, IllegalAccessException { + void doExecuteTest_with_when_condition_should_only_enrich_events_that_match_when_condition() { final String whenCondition = "/peer/status == success"; when(geoIPProcessorConfig.getEntries()).thenReturn(List.of(entry)); when(geoIPProcessorConfig.getWhenCondition()).thenReturn(whenCondition); when(entry.getSource()).thenReturn("/peer/ip"); when(entry.getTarget()).thenReturn(TARGET); - when(entry.getFields()).thenReturn(setFields()); + when(entry.getIncludeFields()).thenReturn(setFields()); final GeoIPProcessor geoIPProcessor = createObjectUnderTest(); @@ -131,11 +160,11 @@ void doExecuteTest_with_when_condition_should_only_enrich_events_that_match_when } @Test - void doExecuteTest_should_add_geo_data_to_event_if_source_is_non_null() throws NoSuchFieldException, IllegalAccessException { + void doExecuteTest_should_add_geo_data_to_event_if_source_is_non_null() { when(geoIPProcessorConfig.getEntries()).thenReturn(List.of(entry)); when(entry.getSource()).thenReturn(SOURCE); when(entry.getTarget()).thenReturn(TARGET); - when(entry.getFields()).thenReturn(setFields()); + when(entry.getIncludeFields()).thenReturn(setFields()); final GeoIPProcessor geoIPProcessor = createObjectUnderTest(); @@ -144,16 +173,75 @@ void doExecuteTest_should_add_geo_data_to_event_if_source_is_non_null() throws N for (final Record record : records) { final Event event = record.getData(); assertThat(event.get("/peer/ip", String.class), equalTo("136.226.242.205")); - assertThat(event.containsKey("geolocation"), equalTo(true)); + assertThat(event.containsKey(TARGET), equalTo(true)); verify(geoIpEventsProcessed).increment(); } } @Test - void doExecuteTest_should_not_add_geo_data_to_event_if_source_is_null() throws NoSuchFieldException, IllegalAccessException { + void doExecuteTest_should_add_geo_data_with_expected_fields_to_event_when_include_fields_is_configured() { + when(geoIPProcessorConfig.getEntries()).thenReturn(List.of(entry)); + when(entry.getSource()).thenReturn(SOURCE); + when(entry.getTarget()).thenReturn(TARGET); + + final List includeFields = List.of("city_name", "asn"); + final List includeFieldsResult = List.of(GeoIPField.CITY_NAME, GeoIPField.ASN); + when(entry.getIncludeFields()).thenReturn(includeFields); + + final GeoIPProcessor geoIPProcessor = createObjectUnderTest(); + + when(geoIPDatabaseReader.getGeoData(any(), any(), any())).thenReturn(prepareGeoData()); + Collection> records = geoIPProcessor.doExecute(setEventQueue()); + verify(geoIPDatabaseReader).getGeoData(any(), geoIPFieldCaptor.capture(), any()); + + for (final Record record : records) { + final Event event = record.getData(); + assertThat(event.get("/peer/ip", String.class), equalTo("136.226.242.205")); + assertThat(event.containsKey(TARGET), equalTo(true)); + verify(geoIpEventsProcessed).increment(); + } + + final List value = geoIPFieldCaptor.getValue(); + assertThat(value, containsInAnyOrder(includeFieldsResult.toArray())); + } + + @Test + void doExecuteTest_should_add_geo_data_with_expected_fields_to_event_when_exclude_fields_is_configured() { + when(geoIPProcessorConfig.getEntries()).thenReturn(List.of(entry)); + when(entry.getSource()).thenReturn(SOURCE); + when(entry.getTarget()).thenReturn(TARGET); + + final List excludeFields = List.of("country_name", "country_iso_code", "city_name", "asn", "asn_organization", "network", "ip"); + final List excludeFieldsResult = List.of(CONTINENT_NAME, CONTINENT_CODE, IS_COUNTRY_IN_EUROPEAN_UNION, + REPRESENTED_COUNTRY_NAME, REPRESENTED_COUNTRY_ISO_CODE, REPRESENTED_COUNTRY_TYPE, REGISTERED_COUNTRY_NAME, + REGISTERED_COUNTRY_ISO_CODE, LOCATION, LOCATION_ACCURACY_RADIUS, LATITUDE, LONGITUDE, METRO_CODE, TIME_ZONE, POSTAL_CODE, + MOST_SPECIFIED_SUBDIVISION_NAME, MOST_SPECIFIED_SUBDIVISION_ISO_CODE, LEAST_SPECIFIED_SUBDIVISION_NAME, + LEAST_SPECIFIED_SUBDIVISION_ISO_CODE, COUNTRY_CONFIDENCE, CITY_CONFIDENCE, MOST_SPECIFIED_SUBDIVISION_CONFIDENCE, + LEAST_SPECIFIED_SUBDIVISION_CONFIDENCE, POSTAL_CODE_CONFIDENCE); + when(entry.getExcludeFields()).thenReturn(excludeFields); + + final GeoIPProcessor geoIPProcessor = createObjectUnderTest(); + + when(geoIPDatabaseReader.getGeoData(any(), any(), any())).thenReturn(prepareGeoData()); + Collection> records = geoIPProcessor.doExecute(setEventQueue()); + verify(geoIPDatabaseReader).getGeoData(any(), geoIPFieldCaptor.capture(), any()); + + for (final Record record : records) { + final Event event = record.getData(); + assertThat(event.get("/peer/ip", String.class), equalTo("136.226.242.205")); + assertThat(event.containsKey(TARGET), equalTo(true)); + verify(geoIpEventsProcessed).increment(); + } + + final List value = geoIPFieldCaptor.getValue(); + assertThat(value, containsInAnyOrder(excludeFieldsResult.toArray())); + } + + @Test + void doExecuteTest_should_not_add_geo_data_to_event_if_source_is_null() { when(geoIPProcessorConfig.getEntries()).thenReturn(List.of(entry)); when(entry.getSource()).thenReturn("ip"); - when(entry.getFields()).thenReturn(setFields()); + when(entry.getIncludeFields()).thenReturn(setFields()); final GeoIPProcessor geoIPProcessor = createObjectUnderTest(); @@ -169,10 +257,10 @@ void doExecuteTest_should_not_add_geo_data_to_event_if_source_is_null() throws N } @Test - void doExecuteTest_should_not_add_geo_data_to_event_if_returned_data_is_empty() throws NoSuchFieldException, IllegalAccessException { + void doExecuteTest_should_not_add_geo_data_to_event_if_returned_data_is_empty() { when(geoIPProcessorConfig.getEntries()).thenReturn(List.of(entry)); when(entry.getSource()).thenReturn(SOURCE); - when(entry.getFields()).thenReturn(setFields()); + when(entry.getIncludeFields()).thenReturn(setFields()); final GeoIPProcessor geoIPProcessor = createObjectUnderTest(); @@ -203,14 +291,14 @@ void doExecuteTest_should_not_add_geodata_if_database_is_expired() { } @Test - void doExecuteTest_should_not_add_geodata_if_database_is_expired_() { + void doExecuteTest_should_not_add_geodata_if_ip_address_is_not_public() { try (final MockedStatic ipValidationCheckMockedStatic = mockStatic(IPValidationCheck.class)) { ipValidationCheckMockedStatic.when(() -> IPValidationCheck.isPublicIpAddress(any())).thenReturn(false); } when(geoIPProcessorConfig.getEntries()).thenReturn(List.of(entry)); when(entry.getSource()).thenReturn(SOURCE); - when(entry.getFields()).thenReturn(setFields()); + when(entry.getIncludeFields()).thenReturn(setFields()); final GeoIPProcessor geoIPProcessor = createObjectUnderTest(); @@ -227,7 +315,7 @@ void doExecuteTest_should_not_add_geodata_if_database_is_expired_() { @Test void test_tags_when_enrich_fails() { when(entry.getSource()).thenReturn(SOURCE); - when(entry.getFields()).thenReturn(setFields()); + when(entry.getIncludeFields()).thenReturn(setFields()); List testTags = List.of("tag1", "tag2"); when(geoIPProcessorConfig.getTagsOnFailure()).thenReturn(testTags); @@ -268,10 +356,10 @@ private Map prepareGeoData() { return geoDataMap; } - private List setFields() { - final List attributes = new ArrayList<>(); - attributes.add(GeoIPField.CITY_NAME); - attributes.add(GeoIPField.COUNTRY_NAME); + private List setFields() { + final List attributes = new ArrayList<>(); + attributes.add("city_name"); + attributes.add("country_name"); return attributes; } diff --git a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/configuration/EntryConfigTest.java b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/configuration/EntryConfigTest.java index d9dd975566..8b095ffb63 100644 --- a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/configuration/EntryConfigTest.java +++ b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/configuration/EntryConfigTest.java @@ -7,14 +7,11 @@ import org.junit.jupiter.api.Test; -import org.opensearch.dataprepper.plugins.processor.GeoIPField; import org.opensearch.dataprepper.test.helper.ReflectivelySetField; -import java.util.Collections; import java.util.List; import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; @@ -32,7 +29,8 @@ void testDefaultConfig() { assertThat(entryConfig.getSource(), is(nullValue())); assertThat(entryConfig.getTarget(), equalTo(DEFAULT_TARGET)); - assertThat(entryConfig.getFields(), is(Collections.emptyList())); + assertThat(entryConfig.getIncludeFields(), equalTo(null)); + assertThat(entryConfig.getExcludeFields(), equalTo(null)); } @Test @@ -41,69 +39,18 @@ void testCustomConfig() throws NoSuchFieldException, IllegalAccessException { final String sourceValue = "source"; final String targetValue = "target"; - final List fieldsValue = List.of(GeoIPField.CITY_NAME, GeoIPField.CONTINENT_CODE); + final List includeFieldsValue = List.of("city_name"); + final List excludeFieldsValue = List.of("asn"); ReflectivelySetField.setField(EntryConfig.class, entryConfig, "source", sourceValue); ReflectivelySetField.setField(EntryConfig.class, entryConfig, "target", targetValue); - ReflectivelySetField.setField(EntryConfig.class, entryConfig, "geoIPFields", fieldsValue); + ReflectivelySetField.setField(EntryConfig.class, entryConfig, "includeFields", includeFieldsValue); + ReflectivelySetField.setField(EntryConfig.class, entryConfig, "excludeFields", excludeFieldsValue); assertThat(entryConfig.getSource(), equalTo(sourceValue)); assertThat(entryConfig.getTarget(), equalTo(targetValue)); - assertThat(entryConfig.getFields(), equalTo(fieldsValue)); - } - - @Test - void test_getFields_with_include_fields_should_return_correct_geoip_fields() throws NoSuchFieldException, IllegalAccessException { - final EntryConfig entryConfig = createObjectUnderTest(); - final List includeFields = List.of("city_name", "continent_code"); - final List fieldsValue = List.of(GeoIPField.CITY_NAME, GeoIPField.CONTINENT_CODE); - - ReflectivelySetField.setField(EntryConfig.class, entryConfig, "includeFields", includeFields); - - assertThat(entryConfig.getFields().size(), equalTo(fieldsValue.size())); - assertThat(entryConfig.getFields(), containsInAnyOrder(fieldsValue.toArray())); - } - - @Test - void test_getFields_with_exclude_keys_should_return_correct_geoip_fields() throws NoSuchFieldException, IllegalAccessException { - final EntryConfig entryConfig = createObjectUnderTest(); - final List excludeFields = List.of("city_name", "continent_code", "continent_name", "is_country_in_european_union", - "asn", "asn_organization", "network", "ip"); - final List fieldsValue = List.of(GeoIPField.LATITUDE, - GeoIPField.REPRESENTED_COUNTRY_ISO_CODE, GeoIPField.LONGITUDE, GeoIPField.COUNTRY_NAME, - GeoIPField.COUNTRY_ISO_CODE, GeoIPField.REGISTERED_COUNTRY_ISO_CODE, GeoIPField.REGISTERED_COUNTRY_NAME, - GeoIPField.COUNTRY_CONFIDENCE, GeoIPField.REPRESENTED_COUNTRY_TYPE, GeoIPField.REPRESENTED_COUNTRY_NAME, - GeoIPField.CITY_CONFIDENCE, GeoIPField.LOCATION, GeoIPField.LOCATION_ACCURACY_RADIUS, GeoIPField.POSTAL_CODE, - GeoIPField.METRO_CODE, GeoIPField.TIME_ZONE, GeoIPField.POSTAL_CODE_CONFIDENCE, GeoIPField.MOST_SPECIFIED_SUBDIVISION_NAME, - GeoIPField.MOST_SPECIFIED_SUBDIVISION_CONFIDENCE, GeoIPField.MOST_SPECIFIED_SUBDIVISION_ISO_CODE, - GeoIPField.LEAST_SPECIFIED_SUBDIVISION_CONFIDENCE, GeoIPField.LEAST_SPECIFIED_SUBDIVISION_ISO_CODE, - GeoIPField.LEAST_SPECIFIED_SUBDIVISION_NAME); - - ReflectivelySetField.setField(EntryConfig.class, entryConfig, "excludeFields", excludeFields); - - assertThat(entryConfig.getFields().size(), equalTo(fieldsValue.size())); - assertThat(entryConfig.getFields(), containsInAnyOrder(fieldsValue.toArray())); - } - - @Test - void test_getFields_with_empty_exclude_keys_should_return_all_geoip_fields() throws NoSuchFieldException, IllegalAccessException { - final EntryConfig entryConfig = createObjectUnderTest(); - final List excludeFields = Collections.emptyList(); - final List fieldsValue = List.of(GeoIPField.LATITUDE, GeoIPField.CITY_NAME, GeoIPField.CONTINENT_CODE, GeoIPField.CONTINENT_NAME, - GeoIPField.IS_COUNTRY_IN_EUROPEAN_UNION, GeoIPField.ASN, GeoIPField.ASN_ORGANIZATION, GeoIPField.NETWORK, GeoIPField.IP, - GeoIPField.REPRESENTED_COUNTRY_ISO_CODE, GeoIPField.LONGITUDE, GeoIPField.COUNTRY_NAME, - GeoIPField.COUNTRY_ISO_CODE, GeoIPField.REGISTERED_COUNTRY_ISO_CODE, GeoIPField.REGISTERED_COUNTRY_NAME, - GeoIPField.COUNTRY_CONFIDENCE, GeoIPField.REPRESENTED_COUNTRY_TYPE, GeoIPField.REPRESENTED_COUNTRY_NAME, - GeoIPField.CITY_CONFIDENCE, GeoIPField.LOCATION, GeoIPField.LOCATION_ACCURACY_RADIUS, GeoIPField.POSTAL_CODE, - GeoIPField.METRO_CODE, GeoIPField.TIME_ZONE, GeoIPField.POSTAL_CODE_CONFIDENCE, GeoIPField.MOST_SPECIFIED_SUBDIVISION_NAME, - GeoIPField.MOST_SPECIFIED_SUBDIVISION_CONFIDENCE, GeoIPField.MOST_SPECIFIED_SUBDIVISION_ISO_CODE, - GeoIPField.LEAST_SPECIFIED_SUBDIVISION_CONFIDENCE, GeoIPField.LEAST_SPECIFIED_SUBDIVISION_ISO_CODE, - GeoIPField.LEAST_SPECIFIED_SUBDIVISION_NAME); - - ReflectivelySetField.setField(EntryConfig.class, entryConfig, "excludeFields", excludeFields); - - assertThat(entryConfig.getFields().size(), equalTo(fieldsValue.size())); - assertThat(entryConfig.getFields(), containsInAnyOrder(fieldsValue.toArray())); + assertThat(entryConfig.getIncludeFields(), equalTo(includeFieldsValue)); + assertThat(entryConfig.getExcludeFields(), equalTo(excludeFieldsValue)); } @Test diff --git a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/exception/EngineFailureExceptionTest.java b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/exception/EngineFailureExceptionTest.java new file mode 100644 index 0000000000..b1a6f4b822 --- /dev/null +++ b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/exception/EngineFailureExceptionTest.java @@ -0,0 +1,33 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.processor.exception; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.UUID; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; + +class EngineFailureExceptionTest { + private String message; + @BeforeEach + void setUp() { + message = UUID.randomUUID().toString(); + } + + private EngineFailureException createObjectUnderTest() { + return new EngineFailureException(message); + } + + @Test + void getMessage_returns_message() { + assertThat(createObjectUnderTest().getMessage(), equalTo(message)); + } + + +} \ No newline at end of file diff --git a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/exception/InvalidIPAddressExceptionTest.java b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/exception/InvalidIPAddressExceptionTest.java new file mode 100644 index 0000000000..d1f9e0c517 --- /dev/null +++ b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/exception/InvalidIPAddressExceptionTest.java @@ -0,0 +1,32 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.processor.exception; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.UUID; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; + +class InvalidIPAddressExceptionTest { + private String message; + @BeforeEach + void setUp() { + message = UUID.randomUUID().toString(); + } + + private InvalidIPAddressException createObjectUnderTest() { + return new InvalidIPAddressException(message); + } + + @Test + void getMessage_returns_message() { + assertThat(createObjectUnderTest().getMessage(), equalTo(message)); + } + +} \ No newline at end of file diff --git a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/DBSourceTest.java b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/DBSourceTest.java deleted file mode 100644 index c7fc843485..0000000000 --- a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/DBSourceTest.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.dataprepper.plugins.processor.extension.databasedownload; - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.MockedStatic; -import org.mockito.junit.jupiter.MockitoExtension; - -import java.io.File; - -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.mockStatic; - -@ExtendWith(MockitoExtension.class) -class DBSourceTest { - - private final String outputFilePath = System.getProperty("java.io.tmpdir") + "GeoTest"; - @Mock - private GeoIPFileManager geoIPFileManager; - - @Test - void createFolderIfNotExistTest() { - try (MockedStatic mockedStatic = mockStatic(DBSource.class)) { - mockedStatic.when(() -> DBSource.createFolderIfNotExist(outputFilePath)).thenReturn(new File(outputFilePath)); - File actualFile = new File(outputFilePath); - assertEquals(actualFile, DBSource.createFolderIfNotExist(outputFilePath)); - } - } - - @Test - void deleteDirectoryTest() { - DBSource.createFolderIfNotExist(outputFilePath); - DBSource.createFolderIfNotExist(outputFilePath + File.separator + "GeoIPz"); - DBSource.createFolderIfNotExist(outputFilePath + File.separator + "GeoIPx"); - assertDoesNotThrow(() -> { - geoIPFileManager.deleteDirectory(new File(outputFilePath)); - }); - } -} \ No newline at end of file diff --git a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/GeoIPDatabaseManagerTest.java b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/GeoIPDatabaseManagerTest.java index 48e35f84d2..f19a3ccc82 100644 --- a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/GeoIPDatabaseManagerTest.java +++ b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/GeoIPDatabaseManagerTest.java @@ -184,7 +184,7 @@ void test_initiateDatabaseDownload_with_geoip2_url_should_use_http_download_serv @Test void test_initiateDatabaseDownload_with_geolite2_cdn_should_use_cdn_download_service_and_geolite2_reader() { - try (final MockedConstruction cdnDownloadServiceMockedConstruction = mockConstruction(CDNDownloadService.class, + try (final MockedConstruction cdnDownloadServiceMockedConstruction = mockConstruction(ManifestDownloadService.class, (mock2, context2)-> doNothing().when(mock2).initiateDownload(List.of(CDN_ENDPOINT))); final MockedConstruction geoLite2DatabaseReaderMockedConstruction = mockConstruction(GeoLite2DatabaseReader.class) ) { @@ -204,7 +204,7 @@ void test_initiateDatabaseDownload_with_geolite2_cdn_should_use_cdn_download_ser @Test void test_updateDatabaseReader_with_geolite2_cdn_should_use_cdn_download_service_and_geolite2_reader_and_get_new_reader() { - try (final MockedConstruction cdnDownloadServiceMockedConstruction = mockConstruction(CDNDownloadService.class, + try (final MockedConstruction cdnDownloadServiceMockedConstruction = mockConstruction(ManifestDownloadService.class, (mock2, context2)-> doNothing().when(mock2).initiateDownload(List.of(CDN_ENDPOINT))); final MockedConstruction geoLite2DatabaseReaderMockedConstruction = mockConstruction(GeoLite2DatabaseReader.class) ) { @@ -218,8 +218,8 @@ void test_updateDatabaseReader_with_geolite2_cdn_should_use_cdn_download_service objectUnderTest.updateDatabaseReader(); assertThat(cdnDownloadServiceMockedConstruction.constructed().size(), equalTo(2)); - for (CDNDownloadService cdnDownloadService: cdnDownloadServiceMockedConstruction.constructed()) { - verify(cdnDownloadService).initiateDownload(List.of(CDN_ENDPOINT)); + for (ManifestDownloadService manifestDownloadService : cdnDownloadServiceMockedConstruction.constructed()) { + verify(manifestDownloadService).initiateDownload(List.of(CDN_ENDPOINT)); } assertThat(geoLite2DatabaseReaderMockedConstruction.constructed().size(), equalTo(2)); // verify if first instance is closed diff --git a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/GeoIPFileManagerTest.java b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/GeoIPFileManagerTest.java new file mode 100644 index 0000000000..c7c74acf8d --- /dev/null +++ b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/GeoIPFileManagerTest.java @@ -0,0 +1,36 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.processor.extension.databasedownload; + +import org.junit.jupiter.api.Test; + +import java.io.File; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +class GeoIPFileManagerTest { + private final String outputFilePath = "./src/test/resources/geoip/test"; + + @Test + void createFolderIfNotExistTest() { + final GeoIPFileManager geoIPFileManager = new GeoIPFileManager(); + geoIPFileManager.createDirectoryIfNotExist(outputFilePath); + + final File file = new File(outputFilePath); + assertTrue(file.exists()); + } + + @Test + void deleteDirectoryTest() { + final GeoIPFileManager geoIPFileManager = new GeoIPFileManager(); + geoIPFileManager.createDirectoryIfNotExist(outputFilePath); + + final File file = new File(outputFilePath); + assertTrue(file.isDirectory()); + geoIPFileManager.deleteDirectory(file); + } + +} \ No newline at end of file diff --git a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/CDNDownloadServiceTest.java b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/ManifestDownloadServiceTest.java similarity index 80% rename from data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/CDNDownloadServiceTest.java rename to data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/ManifestDownloadServiceTest.java index a2d31877df..2dfe276104 100644 --- a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/CDNDownloadServiceTest.java +++ b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/extension/databasedownload/ManifestDownloadServiceTest.java @@ -14,16 +14,16 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -class CDNDownloadServiceTest { +class ManifestDownloadServiceTest { private static final String OUTPUT_DIR = "./src/test/resources/geoip"; - private CDNDownloadService createObjectUnderTest() { - return new CDNDownloadService(OUTPUT_DIR); + private ManifestDownloadService createObjectUnderTest() { + return new ManifestDownloadService(OUTPUT_DIR); } @Test void test_with_valid_endpoint_should_download_file() { - final CDNDownloadService objectUnderTest = createObjectUnderTest(); + final ManifestDownloadService objectUnderTest = createObjectUnderTest(); objectUnderTest.initiateDownload(List.of("https://devo.geoip.maps.opensearch.org/v1/mmdb/geolite2-city/manifest.json")); final File file = new File(OUTPUT_DIR + File.separator + "geolite2-city.mmdb"); @@ -36,7 +36,7 @@ void test_with_valid_endpoint_should_download_file() { @Test void test_with_invalid_endpoint_should_throw_exception() { - final CDNDownloadService objectUnderTest = createObjectUnderTest(); + final ManifestDownloadService objectUnderTest = createObjectUnderTest(); assertThrows(DownloadFailedException.class, () -> objectUnderTest .initiateDownload(List.of("https://devo.geoip.maps.opensearch.org/v1/mmdb/geolite2-enterprise/manifest.json"))); } diff --git a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/utils/DbSourceIdentificationTest.java b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/utils/DbSourceIdentificationTest.java index d581bf82e0..64b3a8f24c 100644 --- a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/utils/DbSourceIdentificationTest.java +++ b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/utils/DbSourceIdentificationTest.java @@ -88,7 +88,7 @@ void getDatabasePathTypeTest_CDN() { List databasePath = List.of(CDN_ENDPOINT_HOST); DBSourceOptions dbSourceOptions = DatabaseSourceIdentification.getDatabasePathType(databasePath); Assertions.assertNotNull(dbSourceOptions); - assertThat(dbSourceOptions, equalTo(DBSourceOptions.CDN)); + assertThat(dbSourceOptions, equalTo(DBSourceOptions.HTTP_MANIFEST)); } @Test diff --git a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/utils/IPValidationCheckTest.java b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/utils/IPValidationCheckTest.java index 487f131f81..03dc47aac0 100644 --- a/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/utils/IPValidationCheckTest.java +++ b/data-prepper-plugins/geoip-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/utils/IPValidationCheckTest.java @@ -9,28 +9,29 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.plugins.processor.exception.InvalidIPAddressException; -import java.net.UnknownHostException; +import static org.junit.jupiter.api.Assertions.assertThrows; @ExtendWith(MockitoExtension.class) class IPValidationCheckTest { private static final String PRIVATE_IP_ADDRESS = "192.168.29.233"; private static final String PUBLIC_IP_ADDRESS = "2001:4860:4860::8888"; - private static final String INVALID_IP_ADDRESS = "255.255.255.0"; + private static final String INVALID_IP_ADDRESS = "255.255.255.999"; @Test - void ipValidationcheckTest_positive() throws UnknownHostException { + void ipValidationcheckTest_public() { Assertions.assertTrue(IPValidationCheck.isPublicIpAddress(PUBLIC_IP_ADDRESS)); } @Test - void ipValidationcheckTest_negative() throws UnknownHostException { + void ipValidationcheckTest_negative() { Assertions.assertFalse(IPValidationCheck.isPublicIpAddress(PRIVATE_IP_ADDRESS)); } @Test - void ipValidationcheckTest_invalid() throws UnknownHostException { - Assertions.assertTrue(IPValidationCheck.isPublicIpAddress(INVALID_IP_ADDRESS)); + void ipValidationcheckTest_invalid() { + assertThrows(InvalidIPAddressException.class, () -> IPValidationCheck.isPublicIpAddress(INVALID_IP_ADDRESS)); } } \ No newline at end of file