Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WIP] Update mantis 3rd party dependency specifications #349

Draft
wants to merge 26 commits into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
4408fa0
Update mantis-common 3rd party dependency specifications
mabelbot Feb 28, 2023
091cd5c
Refactor mantis-common libraries.nettyEpoll
mabelbot Feb 28, 2023
05a20f3
Refactor mantis-common by removing ext variable
mabelbot Feb 28, 2023
2d27da4
Refactor mantis-connector-iceberg 3rd party dependency specifications
mabelbot Mar 1, 2023
fce4d3d
Refactor mantis-connector-job 3rd party dependency specifications
mabelbot Mar 1, 2023
ba469cc
Refactor mantis-connector-kafka 3rd party dependency specifications
mabelbot Mar 1, 2023
55fd2b8
Refactor mantis-control-plane-core 3rd party dependency specifications
mabelbot Mar 1, 2023
0140680
Refactor mantis-control-plane-server 3rd party dependency specifications
mabelbot Mar 1, 2023
8c5929d
Refactor by merging iceberg and akkaHttp deps
mabelbot Mar 1, 2023
34a352e
Refactor mantis-network 3rd party dependency specifications
mabelbot Mar 1, 2023
4926b28
Refactor mantis-publish-core 3rd party dependency specifications
mabelbot Mar 2, 2023
a5a1161
Refactor mantis-publish-netty 3rd party dependency specifications
mabelbot Mar 2, 2023
ea7a19f
Refactor mantis-publish-netty-guice 3rd party dependency specifications
mabelbot Mar 2, 2023
60d8cf2
Refactor mantis-remote-observable 3rd party dependency specifications
mabelbot Mar 2, 2023
42e0ccb
Refactor mantis-server-worker 3rd party dependency specifications
mabelbot Mar 2, 2023
1ddb23d
Refactor mantis-server-worker-client 3rd party dependency specifications
mabelbot Mar 2, 2023
3f1770b
Refactor mantis-shaded 3rd party dependency specifications
mabelbot Mar 2, 2023
ee2a3b4
Refactor mantis-source-job-kafka 3rd party dependency specifications
mabelbot Mar 2, 2023
4a44dfc
Refactor mantis-examples-groupby-sample 3rd party dependency specific…
mabelbot Mar 2, 2023
f74ba90
Refactor mantis-examples-jobconnector-sample 3rd party dependency spe…
mabelbot Mar 2, 2023
bc0ad96
Refactor mantis-examples-mantis-publish-sample 3rd party dependency s…
mabelbot Mar 2, 2023
d90bad4
Refactor mantis-examples-mantis-publish-web-sample 3rd party dependen…
mabelbot Mar 2, 2023
30a65ce
Refactor mantis-examples-sine-function 3rd party dependency specifica…
mabelbot Mar 2, 2023
720f4cd
Refactor mantis-examples-synthetic-sourcejob 3rd party dependency spe…
mabelbot Mar 2, 2023
4c82455
Refactor mantis-examples-twitter-sample 3rd party dependency specific…
mabelbot Mar 2, 2023
100f073
Refactor mantis-examples-wordcount 3rd party dependency specifications
mabelbot Mar 2, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -35,19 +35,31 @@ buildscript {
}

ext.versions = [
archaiusVersion: "latest.release",
avroVersion : "1.10.2",
flink : "1.14.2",
gsonVersion : "2.8.+",
hadoop : "2.7.7",
icebergVersion : "0.14.+",
jctoolsVersion : "1.+",
junit4 : "4.11",
junit5 : "5.4.+",
kafkaVersion : "2.2.+",
mockito : "2.0.+",
mockito3 : "3.+",
nettyVersion : "4.1.34.Final",
parquetVersion : "1.12.0",
snappyVersion : "1.1.+",
spectator: "1.3.+",
slf4j : "1.7.0",
vavr : "0.9.2",
]

ext.libraries = [
archaius2Api : "com.netflix.archaius:archaius2-api:${versions.archaiusVersion}",
archaius2Core : "com.netflix.archaius:archaius2-core:${versions.archaiusVersion}",
asyncHttpClient: "org.asynchttpclient:async-http-client:2.12.3",
avro : "org.apache.avro:avro:${versions.avroVersion}",
commonsIo : "commons-io:commons-io:2.11.0",
commonsLang3 : 'org.apache.commons:commons-lang3:3.5',
flinkCore : "org.apache.flink:flink-core:${versions.flink}",
Expand All @@ -61,27 +73,43 @@ ext.libraries = [
"junit:junit:${versions.junit4}",
"junit:junit-dep:${versions.junit4}",
],
gson : "com.google.code.gson:gson:${versions.gsonVersion}",
hadoopCommon : "org.apache.hadoop:hadoop-common:${versions.hadoop}",
hadoopS3 : "org.apache.hadoop:hadoop-aws:${versions.hadoop}",
hamcrest : "org.hamcrest:hamcrest-core:1.3",
icebergApi : "org.apache.iceberg:iceberg-api:${versions.icebergVersion}",
icebergCore : "org.apache.iceberg:iceberg-core:${versions.icebergVersion}",
icebergData : "org.apache.iceberg:iceberg-data:${versions.icebergVersion}",
icebergParquet : "org.apache.iceberg:iceberg-parquet:${versions.icebergVersion}",
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would merge these four as well into a single dependency.

jcip : "net.jcip:jcip-annotations:1.0",
jctools : "org.jctools:jctools-core:${versions.jctoolsVersion}",
junitJupiter : [
"org.junit.jupiter:junit-jupiter-api:${versions.junit5}",
"org.junit.jupiter:junit-jupiter-engine:${versions.junit5}",
"org.junit.jupiter:junit-jupiter-params:${versions.junit5}",
],
kafkaClients : "org.apache.kafka:kafka-clients:${versions.kafkaVersion}",
mantisShaded : "io.mantisrx:mantis-shaded:2.0.2",
mockitoAll : "org.mockito:mockito-all:${versions.mockito}",
mockitoCore : "org.mockito:mockito-core:${versions.mockito}",
mockitoCore3 : "org.mockito:mockito-core:${versions.mockito3}",
mockneat : "net.andreinc:mockneat:0.4.8",
nettyBuffer : "io.netty:netty-buffer:${versions.nettyVersion}",
nettyCodec : "io.netty:netty-codec-http:${versions.nettyVersion}",
nettyEpoll : "io.netty:netty-transport-native-epoll:${versions.nettyVersion}:linux-x86_64",
parquetHadoop : "org.apache.parquet:parquet-hadoop:${versions.parquetVersion}",
rxJava : "io.reactivex:rxjava:1.3.8",
rxNettyShaded : "com.netflix:mantis-rxnetty:0.4.19.1",
slf4jApi : "org.slf4j:slf4j-api:${versions.slf4j}",
slf4jLog4j12 : "org.slf4j:slf4j-log4j12:${versions.slf4j}",
snappyJava : "org.xerial.snappy:snappy-java:${versions.snappyVersion}",
spectatorApi : "com.netflix.spectator:spectator-api:${versions.spectator}",
spectatorApiv0 : "com.netflix.spectator:spectator-api:0.82.+",
spotifyFutures : "com.spotify:completable-futures:0.3.1",
vavr : "io.vavr:vavr:${versions.vavr}",
vavrJackson : "io.vavr:vavr-jackson:${versions.vavr}",
vavrTest : "io.vavr:vavr-test:${versions.vavr}",
wiremock : "com.github.tomakehurst:wiremock-jre8:2.21.0",
zip4j : "net.lingala.zip4j:zip4j:2.9.0",
]

Expand Down
20 changes: 7 additions & 13 deletions mantis-common/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -16,19 +16,13 @@

apply plugin: 'java-test-fixtures'

ext {
jctoolsVersion = '1.+'
nettyVersion = '4.1.17.Final'
snappyVersion = '1.1.+'
}

dependencies {
api project(":mantis-common-serde")
api "io.netty:netty-codec-http:$nettyVersion"
api "io.netty:netty-buffer:$nettyVersion"
api group: 'io.netty', name: 'netty-transport-native-epoll', classifier: 'linux-x86_64', version: nettyVersion
api "org.xerial.snappy:snappy-java:$snappyVersion"
api "org.jctools:jctools-core:$jctoolsVersion"
api libraries.nettyCodec
api libraries.nettyBuffer
api libraries.nettyEpoll
api libraries.snappyJava
api libraries.jctools

// spectatorApi should be packaged at entry point level to avoid version conflicts.
compileOnly libraries.spectatorApi
Expand All @@ -39,11 +33,11 @@ dependencies {
api libraries.slf4jApi
api libraries.slf4jLog4j12
implementation libraries.commonsIo
implementation 'net.jcip:jcip-annotations:1.0'
implementation libraries.jcip

testImplementation libraries.spectatorApi
testImplementation libraries.commonsLang3
testImplementation "org.hamcrest:hamcrest-core:1.3"
testImplementation libraries.hamcrest
testImplementation libraries.junit4
testImplementation libraries.mockitoCore
}
26 changes: 10 additions & 16 deletions mantis-connectors/mantis-connector-iceberg/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,6 @@
apply plugin: 'com.github.johnrengelman.shadow'
apply plugin: 'mantis'

ext {
hadoopVersion = '2.7.3'
icebergVersion = '0.14.+'
avroVersion = '1.10.2'
parquetVersion = '1.12.0'
}

dependencies {
configurations {
Expand All @@ -33,30 +27,30 @@ dependencies {
// we want a parquet version above 1.12.0 because we need this fix
// https://issues.apache.org/jira/browse/PARQUET-1851
resolutionStrategy {
force "org.apache.parquet:parquet-hadoop:${parquetVersion}"
force libraries.parquetHadoop
}
}
}
implementation project(":mantis-runtime")
implementation libraries.slf4jApi

// We only need the Configuration interface. Users can bring their own hadoop-common version.
shadow "org.apache.hadoop:hadoop-common:$hadoopVersion"
shadow libraries.hadoopCommon

// Exclude in case there are differences in SHAs between 0.7.0 incubation versions.
shadow "org.apache.iceberg:iceberg-api:$icebergVersion"
shadow "org.apache.iceberg:iceberg-core:$icebergVersion"
shadow "org.apache.iceberg:iceberg-data:$icebergVersion"
shadow "org.apache.iceberg:iceberg-parquet:$icebergVersion"
shadow "org.apache.avro:avro:$avroVersion"
shadow "org.apache.parquet:parquet-hadoop:${parquetVersion}"
shadow libraries.icebergApi
shadow libraries.icebergCore
shadow libraries.icebergData
shadow libraries.icebergParquet
shadow libraries.avro
shadow libraries.parquetHadoop

// We only need this for local mains(). Users bring their own implementation.
shadow libraries.slf4jLog4j12

testImplementation project(":mantis-runtime").sourceSets.test.output
testImplementation "org.apache.hadoop:hadoop-common:$hadoopVersion"
testImplementation "org.apache.iceberg:iceberg-data:$icebergVersion"
testImplementation libraries.hadoopCommon
testImplementation libraries.icebergData
testImplementation libraries.junitJupiter
testImplementation libraries.mockitoCore
testImplementation libraries.spectatorApi
Expand Down
5 changes: 1 addition & 4 deletions mantis-connectors/mantis-connector-job/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -16,16 +16,13 @@

apply plugin: 'mantis'

ext {
gsonVersion = '2.8.+'
}

dependencies {
implementation project(":mantis-runtime")
implementation project(":mantis-client")
implementation project(":mantis-control-plane:mantis-control-plane-core")
implementation project(":mantis-publish:mantis-publish-core")

implementation "com.google.code.gson:gson:$gsonVersion"
implementation libraries.gson
implementation libraries.vavr
}
17 changes: 6 additions & 11 deletions mantis-connectors/mantis-connector-kafka/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -16,26 +16,21 @@

apply plugin: 'mantis'

ext {
archaiusVersion = '2.3.+'
spectatorVersion = '0.82.+'
kafkaVersion = '2.2.+'
}

dependencies {
implementation libraries.mantisShaded
implementation project(":mantis-runtime")

api "org.apache.kafka:kafka-clients:$kafkaVersion"
api libraries.kafkaClients

api libraries.rxJava
api "com.netflix.spectator:spectator-api:$spectatorVersion"
api "com.netflix.archaius:archaius2-api:$archaiusVersion"
api "com.netflix.archaius:archaius2-core:$archaiusVersion"
api libraries.spectatorApiv0
api libraries.archaius2Api
api libraries.archaius2Core

testImplementation libraries.junit4
testImplementation project(':mantis-runtime').sourceSets.test.output
testImplementation libraries.mockitoAll
testImplementation "com.github.tomakehurst:wiremock-jre8:2.21.0"
testCompileOnly "com.netflix.archaius:archaius2-core:$archaiusVersion"
testImplementation libraries.wiremock
testCompileOnly libraries.archaius2Core
}