Add geoip service extension #3944
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.dataprepper.plugins.processor.extension; | ||
|
|
||
| import org.opensearch.dataprepper.plugins.processor.GeoIPProcessorService; | ||
|
|
||
| public class DefaultGeoIpConfigSupplier implements GeoIpConfigSupplier { | ||
| private final GeoIpServiceConfig geoIpServiceConfig; | ||
|
|
||
| public DefaultGeoIpConfigSupplier(final GeoIpServiceConfig geoIpServiceConfig) { | ||
| this.geoIpServiceConfig = geoIpServiceConfig; | ||
| } | ||
|
|
||
| @Override | ||
| public GeoIPProcessorService getGeoIPProcessorService() { | ||
| //TODO: use GeoIpServiceConfig and return GeoIPProcessorService | ||
| return null; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.dataprepper.plugins.processor.extension; | ||
|
|
||
| import org.opensearch.dataprepper.model.annotations.DataPrepperExtensionPlugin; | ||
| import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; | ||
| import org.opensearch.dataprepper.model.plugin.ExtensionPlugin; | ||
| import org.opensearch.dataprepper.model.plugin.ExtensionPoints; | ||
|
|
||
| @DataPrepperExtensionPlugin(modelType = GeoIpServiceConfig.class, rootKeyJsonPath = "/geoip_service") | ||
| public class GeoIpConfigExtension implements ExtensionPlugin { | ||
| private final DefaultGeoIpConfigSupplier defaultGeoIpConfigSupplier; | ||
|
|
||
| @DataPrepperPluginConstructor | ||
| public GeoIpConfigExtension(final GeoIpServiceConfig geoIpServiceConfig) { | ||
| this.defaultGeoIpConfigSupplier = new DefaultGeoIpConfigSupplier(geoIpServiceConfig != null ? geoIpServiceConfig : new GeoIpServiceConfig()); | ||
| } | ||
|
|
||
| @Override | ||
| public void apply(final ExtensionPoints extensionPoints) { | ||
| extensionPoints.addExtensionProvider(new GeoIpConfigProvider(this.defaultGeoIpConfigSupplier)); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.dataprepper.plugins.processor.extension; | ||
|
|
||
| import org.opensearch.dataprepper.model.plugin.ExtensionProvider; | ||
|
|
||
| import java.util.Optional; | ||
|
|
||
| public class GeoIpConfigProvider implements ExtensionProvider<GeoIpConfigSupplier> { | ||
| private final GeoIpConfigSupplier geoIpConfigSupplier; | ||
|
|
||
| public GeoIpConfigProvider(final GeoIpConfigSupplier geoIpConfigSupplier) { | ||
| this.geoIpConfigSupplier = geoIpConfigSupplier; | ||
| } | ||
|
|
||
| @Override | ||
| public Optional<GeoIpConfigSupplier> provideInstance(Context context) { | ||
| return Optional.of(this.geoIpConfigSupplier); | ||
| } | ||
|
|
||
| @Override | ||
| public Class<GeoIpConfigSupplier> supportedClass() { | ||
| return GeoIpConfigSupplier.class; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.dataprepper.plugins.processor.extension; | ||
|
|
||
| import org.opensearch.dataprepper.plugins.processor.GeoIPProcessorService; | ||
|
|
||
| /** | ||
| * Interface for supplying {@link GeoIPProcessorService} to {@link GeoIpConfigExtension} | ||
| * | ||
| * @since 2.7 | ||
| */ | ||
| public interface GeoIpConfigSupplier { | ||
| /** | ||
| * Returns the {@link GeoIPProcessorService} | ||
| * | ||
| * @since 2.7 | ||
| */ | ||
| GeoIPProcessorService getGeoIPProcessorService(); | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.dataprepper.plugins.processor.extension; | ||
|
|
||
| import com.fasterxml.jackson.annotation.JsonProperty; | ||
| import jakarta.validation.Valid; | ||
|
|
||
| public class GeoIpServiceConfig { | ||
| private static final MaxMindConfig DEFAULT_MAXMIND_CONFIG = new MaxMindConfig(); | ||
|
|
||
| public GeoIpServiceConfig() { | ||
| // This default constructor is used if geoip_service is not configured | ||
| } | ||
|
|
||
| @JsonProperty("maxmind") | ||
| @Valid | ||
| private MaxMindConfig maxMindConfig = DEFAULT_MAXMIND_CONFIG; | ||
|
|
||
| /** | ||
| * Gets the configuration for MaxMind. | ||
| * | ||
| * @return The MaxMind configuration | ||
| * @since 2.7 | ||
| */ | ||
| public MaxMindConfig getMaxMindConfig() { | ||
| return maxMindConfig; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,99 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.dataprepper.plugins.processor.extension; | ||
|
|
||
| import com.fasterxml.jackson.annotation.JsonProperty; | ||
| import jakarta.validation.Valid; | ||
| import jakarta.validation.constraints.AssertTrue; | ||
| import jakarta.validation.constraints.Min; | ||
| import org.hibernate.validator.constraints.time.DurationMax; | ||
| import org.hibernate.validator.constraints.time.DurationMin; | ||
|
|
||
| import java.time.Duration; | ||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
|
|
||
| public class MaxMindConfig { | ||
| private static final String S3_PREFIX = "s3://"; | ||
|
|
||
| //TODO: Add validations to database paths | ||
| //TODO: Make default path to be a public CDN endpoint | ||
| private static final List<String> DEFAULT_DATABASE_PATHS = new ArrayList<>(); | ||
| private static final Duration DEFAULT_DATABASE_REFRESH_INTERVAL = Duration.ofDays(7); | ||
| private static final int DEFAULT_CACHE_SIZE = 4096; | ||
|
|
||
| @JsonProperty("database_paths") | ||
| private List<String> databasePaths = DEFAULT_DATABASE_PATHS; | ||
|
|
||
| @JsonProperty("database_refresh_interval") | ||
| @DurationMin(days = 1) | ||
| @DurationMax(days = 30) | ||
| private Duration databaseRefreshInterval = DEFAULT_DATABASE_REFRESH_INTERVAL; | ||
|
|
||
| @JsonProperty("cache_size") | ||
| @Min(1) | ||
| //TODO: Add a Max limit on cache size | ||
| private int cacheSize = DEFAULT_CACHE_SIZE; | ||
|
|
||
| //TODO: Add a destination path to store database files | ||
|
There was a problem hiding this comment. Not following this comment, are you intended for storing files in database_paths into different paths? There was a problem hiding this comment. Yeah, right now it's stored in a temp folder. But it should be configured |
||
| @JsonProperty("aws") | ||
| @Valid | ||
| private AwsAuthenticationOptionsConfig awsAuthenticationOptionsConfig; | ||
|
|
||
| public MaxMindConfig() { | ||
| // This default constructor is used if maxmind is not configured | ||
| } | ||
|
|
||
| @AssertTrue(message = "aws should be configured if any path in database_paths is S3 bucket path.") | ||
| boolean isAwsAuthenticationOptionsRequired() { | ||
| for (final String databasePath : databasePaths) { | ||
| if (databasePath.startsWith(S3_PREFIX)) { | ||
| return awsAuthenticationOptionsConfig != null; | ||
| } | ||
| } | ||
| return true; | ||
| } | ||
|
|
||
| /** | ||
| * Gets the MaxMind database paths | ||
| * | ||
| * @return The MaxMind database paths | ||
| * @since 2.7 | ||
| */ | ||
| public List<String> getDatabasePaths() { | ||
|
There was a problem hiding this comment. Why do we have multiple paths? Is this to include the country, ASN, etc.? There was a problem hiding this comment. Yeah, for multiple databases city, country, ASN. I will add an annotation so it can have max size of 3. |
||
| return databasePaths; | ||
| } | ||
|
|
||
| /** | ||
| * Gets the database refresh duration. This loads the database from the paths into memory again in case if there are any updates. | ||
| * | ||
| * @return The refresh duration | ||
| * @since 2.7 | ||
| */ | ||
| public Duration getDatabaseRefreshInterval() { | ||
| return databaseRefreshInterval; | ||
| } | ||
|
|
||
| /** | ||
| * Gets the cache size used in CHM cache for MaxMind DatabaseReader. | ||
| * | ||
| * @return The cache size | ||
| * @since 2.7 | ||
| */ | ||
| public int getCacheSize() { | ||
| return cacheSize; | ||
| } | ||
|
|
||
| /** | ||
| * Gets the AWS authentication config used for reading from S3 bucket | ||
| * | ||
| * @return The AWS authentication options | ||
| * @since 2.7 | ||
| */ | ||
| public AwsAuthenticationOptionsConfig getAwsAuthenticationOptionsConfig() { | ||
| return awsAuthenticationOptionsConfig; | ||
| } | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Do you have any thoughts on the maximum cache size? How big is each entry?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I don't have any right now, but it's something to think about as it can abused if max limit is not added