Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Integrate CDC data from MySQL/Postgres/MongoDb data source #3313

Merged
merged 30 commits into from
Oct 12, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
7ac1e3c
Add KafkaClusterExtensionWithConfig plugin
wanghd89 Sep 4, 2023
3b4e25e
add unit tests to kafka cluster config extension
Sep 4, 2023
7286de8
integrate with kafka connect
Sep 4, 2023
5e373ed
Modify Config, waiting for final confirm of Config files' change
Sep 5, 2023
587a3d5
update config to make mysql/postgres/mongodb as source
Sep 6, 2023
eabc079
add ExtractNewRecordState transforms to simplify the CDC data
Sep 7, 2023
e4d77d7
fix mysql postgres field
Sep 7, 2023
cd8ce61
update readme
Sep 7, 2023
74102d2
add missing unit test for PluginMetrics.
Sep 8, 2023
2c0adf1
fix unit test and add snapshot_fetch_size for mongoDB to increase sna…
Sep 11, 2023
8219819
address PR comments to use Duration
Sep 12, 2023
5f7f502
fix checkstyle
Sep 12, 2023
27d1693
rename config and exclude zookeeper according to PR review
Sep 14, 2023
f5b66c9
Merge remote-tracking branch 'upstream/main' into feat-cdc
Sep 15, 2023
71ed291
expose connector_rebalance_max_delay config to customers. The default…
Sep 15, 2023
dc9ada6
address comments from PR
Sep 18, 2023
fcd1708
Merge branch 'main' of github.com:opensearch-project/data-prepper int…
Sep 28, 2023
1b20224
fix merge conflicts
Sep 28, 2023
74f0f1d
minimize affects the jetty-boom version
Sep 28, 2023
af526c9
revert the change
Sep 28, 2023
b6d1ad0
Merge remote-tracking branch 'upstream/main' into feat-cdc
Oct 7, 2023
926dd1a
merge mainline into feat-cdc-kafkaconnect branch. Resolved conflicts
Oct 7, 2023
45d97ae
resolve jetty CVE check by using latest 9.x version. revert using pla…
Oct 7, 2023
401acb4
downgrade scoped Jetty dependency to use 9.4.52 version for Kafka-Con…
Oct 7, 2023
3dcb858
remove kotlin-compiler to fix service_map plugin
Oct 9, 2023
845949e
Merge remote-tracking branch 'upstream/main' into feat-cdc
Oct 9, 2023
0323ded
fix checkstyle caused by KK
Oct 10, 2023
8f51c45
Merge remote-tracking branch 'upstream/main' into feat-cdc
Oct 11, 2023
1a1dff5
fix CVE
Oct 11, 2023
a9ebf15
update CVE to use latest jetty9.4.53 version
Oct 12, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ subprojects {
}
dependencies {
implementation platform('com.fasterxml.jackson:jackson-bom:2.15.0')
implementation platform('org.eclipse.jetty:jetty-bom:11.0.17')
implementation platform('org.eclipse.jetty:jetty-bom:9.4.53.v20231009')
implementation platform('io.micrometer:micrometer-bom:1.10.5')
implementation libs.guava.core
implementation libs.slf4j.api
Expand Down Expand Up @@ -154,13 +154,13 @@ subprojects {
}
implementation('org.eclipse.jetty:http2-common') {
version {
require '11.0.17'
require '9.4.53.v20231009'
}
because 'Fixes CVE-2023-44487'
}
implementation('org.eclipse.jetty:http2-server') {
version {
require '11.0.17'
require '9.4.53.v20231009'
}
because 'Fixes CVE-2023-44487'
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.DistributionSummary;
import io.micrometer.core.instrument.Metrics;
import io.micrometer.core.instrument.Tag;
import io.micrometer.core.instrument.Timer;
import org.opensearch.dataprepper.model.configuration.PluginSetting;

Expand Down Expand Up @@ -78,8 +79,11 @@ public <T> T gauge(String name, T obj, ToDoubleFunction<T> valueFunction) {
return Metrics.gauge(getMeterName(name), obj, valueFunction);
}

public <T> T gaugeWithTags(String name, Iterable<Tag> tags, T obj, ToDoubleFunction<T> valueFunction) {
return Metrics.gauge(getMeterName(name), tags, obj, valueFunction);
}

private String getMeterName(final String name) {
return new StringJoiner(MetricNames.DELIMITER).add(metricsPrefix).add(name).toString();
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import java.util.StringJoiner;
import java.util.concurrent.atomic.AtomicInteger;

import static java.util.Collections.emptyList;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
Expand Down Expand Up @@ -125,6 +126,18 @@ public void testReferenceGauge() {
assertEquals(3, gauge.length());
}

@Test
public void testReferenceGaugeWithTags() {
final String testString = "abc";
final String gauge = objectUnderTest.gaugeWithTags("gauge", emptyList(), testString, String::length);
assertNotNull(
Metrics.globalRegistry.get(new StringJoiner(MetricNames.DELIMITER)
.add(PIPELINE_NAME).add(PLUGIN_NAME)
.add("gauge").toString()).meter());
assertEquals(3, gauge.length());
}


@Test
public void testEmptyPipelineName() {
assertThrows(
Expand Down
2 changes: 1 addition & 1 deletion data-prepper-core/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ dependencies {
testImplementation project(':data-prepper-plugins:common').sourceSets.test.output
implementation 'com.fasterxml.jackson.core:jackson-databind'
implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml'
implementation "org.reflections:reflections:0.10.2"
implementation libs.reflections.core
implementation 'io.micrometer:micrometer-core'
implementation 'io.micrometer:micrometer-registry-prometheus'
implementation 'io.micrometer:micrometer-registry-cloudwatch2'
Expand Down
2 changes: 1 addition & 1 deletion data-prepper-plugins/common/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ dependencies {
implementation libs.commons.lang3
implementation libs.bouncycastle.bcprov
implementation libs.bouncycastle.bcpkix
implementation 'org.reflections:reflections:0.10.2'
implementation libs.reflections.core
implementation 'io.micrometer:micrometer-core'
testImplementation testLibs.junit.vintage
implementation 'org.apache.parquet:parquet-common:1.13.1'
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
package org.opensearch.dataprepper.plugins.sink;

import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin;
import org.opensearch.dataprepper.model.record.Record;
import org.opensearch.dataprepper.model.sink.Sink;

import java.util.Collection;

@DataPrepperPlugin(name = "noop", pluginType = Sink.class)
public class NoopSink implements Sink<Record<Object>> {
@Override
public void output(Collection<Record<Object>> records) {
// empty by design.
}

@Override
public void shutdown() {
// empty by design.
}

@Override
public void initialize() {
// empty by design.
}

@Override
public boolean isReady() {
return true;
}
}
Loading