Skip to content

Commit

Permalink
Merge pull request #14 from Bit-Quill/revise-flink-dependency-with-sa…
Browse files Browse the repository at this point in the history 
…mple-app

Revise Flink Dependency with Sample Connector Application
  • Loading branch information
@forestmvey
forestmvey authored Jul 3, 2024
2 parents d530850 + 230e261 commit 728a4dc
Show file tree
Hide file tree
Showing 4 changed files with 27 additions and 19 deletions.
3 changes: 2 additions & 1 deletion integrations/flink_connector/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,8 @@ aws s3 cp target/sample-kinesis-to-timestream-app-0.1-SNAPSHOT.jar s3://YOUR_BUC
"Action": [
"kinesis:GetShardIterator",
"kinesis:GetRecords",
"kinesis:ListShards"
"kinesis:ListShards",
"kinesis:DescribeStream"
],
"Resource": "arn:aws:kinesis:<region>:<account-id>:stream/TimestreamTestStream"
}
Expand Down
13 changes: 10 additions & 3 deletions integrations/flink_connector/flink-connector-timestream/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ under the License.
<modelVersion>4.0.0</modelVersion>
<artifactId>flink-connector-timestream</artifactId>
<groupId>com.amazonaws.samples.connectors.timestream</groupId>
<version>0.3-SNAPSHOT</version>
<version>0.5-SNAPSHOT</version>
<packaging>jar</packaging>

<name>flink-connector-timestream</name>
Expand All @@ -33,6 +33,7 @@ under the License.
<guava.version>33.2.1-jre</guava.version>
<java.version>1.11</java.version>
<jdk.version>11</jdk.version>
<awssdk.version>2.26.9</awssdk.version>
<next.jdk.version>12</next.jdk.version>
<maven.compiler.source>${java.version}</maven.compiler.source>
<maven.compiler.target>${java.version}</maven.compiler.target>
Expand All @@ -43,7 +44,7 @@ under the License.
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>bom</artifactId>
<version>2.20.101</version>
<version>${awssdk.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
Expand All @@ -64,6 +65,11 @@ under the License.
<artifactId>flink-connector-base</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-core</artifactId>
<version>${flink.version}</version>
</dependency>

<!-- Flink Test Utils -->
<dependency>
Expand All @@ -85,12 +91,13 @@ under the License.
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>timestreamwrite</artifactId>
<version>${awssdk.version}</version>
</dependency>

<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>aws-crt-client</artifactId>
<version>2.20.101</version>
<version>${awssdk.version}</version>
</dependency>

<dependency>
Expand Down
2 changes: 1 addition & 1 deletion integrations/flink_connector/sample-kinesis-to-timestream-app/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ mvn clean compile && mvn package

You can run the application on your local environment by running `StreamingJobs` class:
```
mvn install exec:java -Dexec.mainClass="com.amazonaws.samples.kinesis2timestream.StreamingJob" -Dexec.args="--InputStreamName TimestreamTestStream --Region us-east-1 --TimestreamDbName kdaflink --TimestreamTableName kinesisdata" -Dexec.classpathScope=test
java -jar target/sample-kinesis-to-timestream-app-0.1-SNAPSHOT.jar --InputStreamName TimestreamTestStream --Region us-east-1 --TimestreamDbName kdaflink --TimestreamTableName kinesisdata
```

In another terminal, concurrently, run the [python sample data generator](../sample-data-generator).
28 changes: 14 additions & 14 deletions integrations/flink_connector/sample-kinesis-to-timestream-app/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,8 @@ under the License.

<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<flink.version>1.16.3</flink.version>
<flink.version>1.18.1</flink.version>
<flink.kinesis.version>4.2.0-1.18</flink.kinesis.version>
<kda.version>2.0.0</kda.version>
<kda.runtime.version>1.2.0</kda.runtime.version>
<java.version>1.11</java.version>
Expand All @@ -38,14 +39,15 @@ under the License.
<maven.compiler.source>${java.version}</maven.compiler.source>
<maven.compiler.target>${java.version}</maven.compiler.target>
<log4j.version>2.17.1</log4j.version>
<slf4j.version>2.0.12</slf4j.version>
</properties>

<dependencyManagement>
<dependencies>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>bom</artifactId>
<version>2.17.203</version>
<version>2.26.9</version>
<type>pom</type>
<scope>import</scope>
</dependency>
Expand All @@ -68,32 +70,32 @@ under the License.

<dependencies>
<!-- Apache Flink dependencies -->
<!-- These dependencies are provided, because they should not be packaged into the JAR file. -->
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-java</artifactId>
<version>${flink.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-java</artifactId>
<version>${flink.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-clients</artifactId>
<version>${flink.version}</version>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-core</artifactId>
<version>${flink.version}</version>
</dependency>
<!-- Add connector dependencies here. They must be in the default scope (compile). -->
<!-- For reading from Kinesis -->
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-connector-kinesis</artifactId>
<version>${flink.version}</version>
<version>${flink.kinesis.version}</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
Expand All @@ -115,13 +117,13 @@ under the License.
<dependency>
<groupId>software.amazon.timestream</groupId>
<artifactId>flink-connector-timestream</artifactId>
<version>0.3</version>
<version>0.5</version>
</dependency>

<!-- sdk dependencies -->
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>timestreamwrite</artifactId>
<version>2.26.9</version>
</dependency>

<!-- Test & compile dependencies -->
Expand Down Expand Up @@ -149,13 +151,12 @@ under the License.
<dependency>
<groupId>org.slf4j</groupId>
<artifactId> slf4j-api</artifactId>
<version>1.7.32</version>
<version>${slf4j.version}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.32</version>
<scope>runtime</scope>
<version>${slf4j.version}</version>
</dependency>
<dependency>
<groupId>de.javakaffee</groupId>
Expand Down Expand Up @@ -233,7 +234,6 @@ under the License.
<excludes>
<exclude>org.apache.flink:force-shading</exclude>
<exclude>com.google.code.findbugs:jsr305</exclude>
<exclude>org.slf4j:*</exclude>
<exclude>log4j:*</exclude>
</excludes>
</artifactSet>
Expand Down

0 comments on commit 728a4dc

Please sign in to comment.