Skip to content

Commit f5cf8e0

Browse files
darenwktdarenwkt
committed
[FLINK-37733] Externalise DynamoDB connector IT Test to E2E test package
1 parent 9d6746b commit f5cf8e0

File tree

13 files changed

+182
-11
lines changed

13 files changed

+182
-11
lines changed

‎flink-connector-aws-e2e-tests/flink-connector-dynamodb-e2e-tests/pom.xml

+123
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
~ Licensed to the Apache Software Foundation (ASF) under one
4+
~ or more contributor license agreements. See the NOTICE file
5+
~ distributed with this work for additional information
6+
~ regarding copyright ownership. The ASF licenses this file
7+
~ to you under the Apache License, Version 2.0 (the
8+
~ "License"); you may not use this file except in compliance
9+
~ with the License. You may obtain a copy of the License at
10+
~
11+
~ http://www.apache.org/licenses/LICENSE-2.0
12+
~
13+
~ Unless required by applicable law or agreed to in writing, software
14+
~ distributed under the License is distributed on an "AS IS" BASIS,
15+
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
~ See the License for the specific language governing permissions and
17+
~ limitations under the License.
18+
-->
19+
20+
<project xmlns="http://maven.apache.org/POM/4.0.0"
21+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
22+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
23+
24+
<parent>
25+
<artifactId>flink-connector-aws-e2e-tests-parent</artifactId>
26+
<groupId>org.apache.flink</groupId>
27+
<version>5.1-SNAPSHOT</version>
28+
</parent>
29+
30+
<modelVersion>4.0.0</modelVersion>
31+
32+
<artifactId>flink-connector-dynamodb-e2e-tests</artifactId>
33+
<name>Flink : Connectors : AWS : E2E Tests : Amazon DynamoDB</name>
34+
<packaging>jar</packaging>
35+
36+
<dependencies>
37+
<dependency>
38+
<groupId>org.apache.flink</groupId>
39+
<artifactId>flink-streaming-java</artifactId>
40+
<version>${flink.version}</version>
41+
<scope>test</scope>
42+
</dependency>
43+
44+
<dependency>
45+
<groupId>org.apache.flink</groupId>
46+
<artifactId>flink-table-api-java</artifactId>
47+
<version>${flink.version}</version>
48+
<scope>test</scope>
49+
</dependency>
50+
51+
<dependency>
52+
<groupId>org.apache.flink</groupId>
53+
<artifactId>flink-table-runtime</artifactId>
54+
<version>${flink.version}</version>
55+
<scope>test</scope>
56+
</dependency>
57+
58+
<dependency>
59+
<groupId>org.apache.flink</groupId>
60+
<artifactId>flink-table-planner-loader</artifactId>
61+
<version>${flink.version}</version>
62+
<scope>test</scope>
63+
</dependency>
64+
65+
<dependency>
66+
<groupId>org.apache.flink</groupId>
67+
<artifactId>flink-connector-dynamodb</artifactId>
68+
<version>${project.version}</version>
69+
<scope>test</scope>
70+
</dependency>
71+
72+
<dependency>
73+
<groupId>org.apache.flink</groupId>
74+
<artifactId>flink-connector-aws-base</artifactId>
75+
<version>${project.version}</version>
76+
<scope>test</scope>
77+
</dependency>
78+
79+
<dependency>
80+
<groupId>org.apache.flink</groupId>
81+
<artifactId>flink-sql-connector-dynamodb</artifactId>
82+
<version>${project.version}</version>
83+
<scope>test</scope>
84+
</dependency>
85+
86+
<!-- Other third-party dependencies -->
87+
<dependency>
88+
<groupId>com.google.guava</groupId>
89+
<artifactId>guava</artifactId>
90+
<scope>test</scope>
91+
</dependency>
92+
93+
<dependency>
94+
<groupId>com.fasterxml.jackson.core</groupId>
95+
<artifactId>jackson-databind</artifactId>
96+
<scope>test</scope>
97+
</dependency>
98+
99+
<dependency>
100+
<groupId>com.fasterxml.jackson.datatype</groupId>
101+
<artifactId>jackson-datatype-jsr310</artifactId>
102+
<scope>test</scope>
103+
</dependency>
104+
105+
<dependency>
106+
<groupId>software.amazon.awssdk</groupId>
107+
<artifactId>s3</artifactId>
108+
<scope>test</scope>
109+
</dependency>
110+
111+
<dependency>
112+
<groupId>software.amazon.awssdk</groupId>
113+
<artifactId>dynamodb</artifactId>
114+
<scope>test</scope>
115+
</dependency>
116+
117+
<dependency>
118+
<groupId>software.amazon.awssdk</groupId>
119+
<artifactId>dynamodb-enhanced</artifactId>
120+
<scope>test</scope>
121+
</dependency>
122+
</dependencies>
123+
</project>

‎flink-connector-aws/flink-connector-dynamodb/src/test/java/org/apache/flink/connector/dynamodb/sink/DynamoDbSinkITCase.java renamed to ‎flink-connector-aws-e2e-tests/flink-connector-dynamodb-e2e-tests/src/test/java/org/apache/flink/connector/dynamodb/sink/test/DynamoDbSinkITCase.java

+20-2
Original file line numberDiff line numberDiff line change
@@ -16,14 +16,19 @@
1616
* limitations under the License.
1717
*/
1818

19-
package org.apache.flink.connector.dynamodb.sink;
19+
package org.apache.flink.connector.dynamodb.sink.test;
2020

2121
import org.apache.flink.api.common.restartstrategy.RestartStrategies;
22+
import org.apache.flink.api.connector.sink2.SinkWriter;
23+
import org.apache.flink.connector.base.sink.writer.ElementConverter;
24+
import org.apache.flink.connector.dynamodb.sink.DynamoDbSink;
25+
import org.apache.flink.connector.dynamodb.sink.DynamoDbWriteRequest;
26+
import org.apache.flink.connector.dynamodb.sink.DynamoDbWriteRequestType;
27+
import org.apache.flink.connector.dynamodb.testutils.DockerImageVersions;
2228
import org.apache.flink.connector.dynamodb.testutils.DynamoDBHelpers;
2329
import org.apache.flink.connector.dynamodb.testutils.DynamoDbContainer;
2430
import org.apache.flink.connector.dynamodb.testutils.Item;
2531
import org.apache.flink.connector.dynamodb.testutils.Items;
26-
import org.apache.flink.connector.dynamodb.util.DockerImageVersions;
2732
import org.apache.flink.runtime.client.JobExecutionException;
2833
import org.apache.flink.streaming.api.datastream.DataStream;
2934
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
@@ -347,4 +352,17 @@ public Scenario withClientProperties(Properties properties) {
347352
return this;
348353
}
349354
}
355+
356+
private static class TestDynamoDbElementConverter
357+
implements ElementConverter<Map<String, AttributeValue>, DynamoDbWriteRequest> {
358+
359+
@Override
360+
public DynamoDbWriteRequest apply(
361+
Map<String, AttributeValue> elements, SinkWriter.Context context) {
362+
return DynamoDbWriteRequest.builder()
363+
.setType(DynamoDbWriteRequestType.PUT)
364+
.setItem(elements)
365+
.build();
366+
}
367+
}
350368
}

‎flink-connector-aws/flink-connector-dynamodb/src/test/java/org/apache/flink/connector/dynamodb/sink/TestRequestMapper.java renamed to ‎flink-connector-aws-e2e-tests/flink-connector-dynamodb-e2e-tests/src/test/java/org/apache/flink/connector/dynamodb/sink/test/TestRequestMapper.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
* limitations under the License.
1717
*/
1818

19-
package org.apache.flink.connector.dynamodb.sink;
19+
package org.apache.flink.connector.dynamodb.sink.test;
2020

2121
import org.apache.flink.api.common.functions.RichMapFunction;
2222

‎flink-connector-aws/flink-connector-dynamodb/src/test/java/org/apache/flink/connector/dynamodb/table/DynamoDbDynamicSinkITCase.java renamed to ‎flink-connector-aws-e2e-tests/flink-connector-dynamodb-e2e-tests/src/test/java/org/apache/flink/connector/dynamodb/table/test/DynamoDbDynamicSinkITCase.java

+4-3
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,13 @@
1616
* limitations under the License.
1717
*/
1818

19-
package org.apache.flink.connector.dynamodb.table;
19+
package org.apache.flink.connector.dynamodb.table.test;
2020

2121
import org.apache.flink.api.common.restartstrategy.RestartStrategies;
22+
import org.apache.flink.connector.dynamodb.table.DynamoDbDynamicSink;
23+
import org.apache.flink.connector.dynamodb.testutils.DockerImageVersions;
2224
import org.apache.flink.connector.dynamodb.testutils.DynamoDBHelpers;
2325
import org.apache.flink.connector.dynamodb.testutils.DynamoDbContainer;
24-
import org.apache.flink.connector.dynamodb.util.DockerImageVersions;
2526
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
2627
import org.apache.flink.table.api.EnvironmentSettings;
2728
import org.apache.flink.table.api.Table;
@@ -49,7 +50,7 @@
4950
import java.util.UUID;
5051
import java.util.concurrent.ExecutionException;
5152

52-
/** Integration test for {@link org.apache.flink.connector.dynamodb.table.DynamoDbDynamicSink}. */
53+
/** Integration test for {@link DynamoDbDynamicSink}. */
5354
@Testcontainers
5455
@ExtendWith(MiniClusterExtension.class)
5556
public class DynamoDbDynamicSinkITCase {

‎flink-connector-aws/flink-connector-dynamodb/src/test/java/org/apache/flink/connector/dynamodb/util/DockerImageVersions.java renamed to ‎flink-connector-aws-e2e-tests/flink-connector-dynamodb-e2e-tests/src/test/java/org/apache/flink/connector/dynamodb/testutils/DockerImageVersions.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
* limitations under the License.
1717
*/
1818

19-
package org.apache.flink.connector.dynamodb.util;
19+
package org.apache.flink.connector.dynamodb.testutils;
2020

2121
/**
2222
* Utility class for defining the image names and versions of Docker containers used during the Java

‎flink-connector-aws/flink-connector-dynamodb/src/test/java/org/apache/flink/connector/dynamodb/testutils/DynamoDBHelpers.java renamed to ‎flink-connector-aws-e2e-tests/flink-connector-dynamodb-e2e-tests/src/test/java/org/apache/flink/connector/dynamodb/testutils/DynamoDBHelpers.java

File renamed without changes.

‎flink-connector-aws/flink-connector-dynamodb/src/test/java/org/apache/flink/connector/dynamodb/testutils/DynamoDbContainer.java renamed to ‎flink-connector-aws-e2e-tests/flink-connector-dynamodb-e2e-tests/src/test/java/org/apache/flink/connector/dynamodb/testutils/DynamoDbContainer.java

File renamed without changes.

‎flink-connector-aws/flink-connector-dynamodb/src/test/java/org/apache/flink/connector/dynamodb/testutils/Item.java renamed to ‎flink-connector-aws-e2e-tests/flink-connector-dynamodb-e2e-tests/src/test/java/org/apache/flink/connector/dynamodb/testutils/Item.java

+3-3
Original file line numberDiff line numberDiff line change
@@ -26,15 +26,15 @@
2626
/** DynamoDB item container. */
2727
public class Item {
2828

29-
public static Item.ItemBuilder builder() {
30-
return new Item.ItemBuilder();
29+
public static ItemBuilder builder() {
30+
return new ItemBuilder();
3131
}
3232

3333
/** Builder to constrict DynamoDB item. */
3434
public static final class ItemBuilder {
3535
Map<String, AttributeValue> item = new HashMap<>();
3636

37-
public Item.ItemBuilder attr(String name, String value) {
37+
public ItemBuilder attr(String name, String value) {
3838
item.put(name, AttributeValue.builder().s(value).build());
3939
return this;
4040
}

‎flink-connector-aws/flink-connector-dynamodb/src/test/java/org/apache/flink/connector/dynamodb/testutils/Items.java renamed to ‎flink-connector-aws-e2e-tests/flink-connector-dynamodb-e2e-tests/src/test/java/org/apache/flink/connector/dynamodb/testutils/Items.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828
public class Items {
2929

3030
public static ItemsBuilder builder() {
31-
return new Items.ItemsBuilder();
31+
return new ItemsBuilder();
3232
}
3333

3434
/** Builder to constrict DynamoDB items. */

‎flink-connector-aws/flink-connector-dynamodb/src/test/resources/create-table.sql renamed to ‎flink-connector-aws-e2e-tests/flink-connector-dynamodb-e2e-tests/src/test/resources/create-table.sql

File renamed without changes.

‎flink-connector-aws/flink-connector-dynamodb/src/test/resources/datagen.sql renamed to ‎flink-connector-aws-e2e-tests/flink-connector-dynamodb-e2e-tests/src/test/resources/datagen.sql

File renamed without changes.

‎flink-connector-aws-e2e-tests/flink-connector-dynamodb-e2e-tests/src/test/resources/log4j2-test.properties

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
################################################################################
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
################################################################################
18+
19+
# Set root logger level to OFF to not flood build logs
20+
# set manually to INFO for debugging purposes
21+
rootLogger.level = OFF
22+
rootLogger.appenderRef.test.ref = TestLogger
23+
24+
appender.testlogger.name = TestLogger
25+
appender.testlogger.type = CONSOLE
26+
appender.testlogger.target = SYSTEM_ERR
27+
appender.testlogger.layout.type = PatternLayout
28+
appender.testlogger.layout.pattern = %-4r [%t] %-5p %c %x - %m%n

‎flink-connector-aws-e2e-tests/pom.xml

+1
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ under the License.
4242
<module>flink-connector-aws-kinesis-streams-e2e-tests</module>
4343
<module>flink-connector-kinesis-e2e-tests</module>
4444
<module>flink-connector-aws-sqs-e2e-tests</module>
45+
<module>flink-connector-dynamodb-e2e-tests</module>
4546
<module>flink-formats-avro-glue-schema-registry-e2e-tests</module>
4647
<module>flink-formats-json-glue-schema-registry-e2e-tests</module>
4748
</modules>

0 commit comments

Comments
 (0)