Skip to content

Commit 8564928

Browse files
authored
1. fix README sink example option 2. update pom.xml for building with JDK9+ (awslabs#6)
1 parent 4815d53 commit 8564928

File tree

3 files changed

+51
-3
lines changed

3 files changed

+51
-3
lines changed

README.md

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -194,9 +194,11 @@ kinesis
194194
.groupBy("data").count()
195195
.writeStream
196196
.format("aws-kinesis")
197-
.outputMode("append")
198-
.option("streamName", "sparkSinkTest")
199-
.option("endpointUrl", "https://kinesis.us-east-1.amazonaws.com")
197+
.outputMode("append")
198+
.option("kinesis.region", "us-east-1")
199+
.option("kinesis.streamName", "sparkSinkTest")
200+
.option("kinesis.endpointUrl", "https://kinesis.us-east-1.amazonaws.com")
201+
.option("checkpointLocation", "/path/to/checkpoint")
200202
.start()
201203
```
202204

pom.xml

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -620,6 +620,42 @@
620620
</plugins>
621621
</build>
622622
</profile>
623+
<profile>
624+
<id>java-8-compatible</id>
625+
<activation>
626+
<jdk>[9,)</jdk>
627+
</activation>
628+
<properties>
629+
<maven.compiler.release>8</maven.compiler.release>
630+
</properties>
631+
<build>
632+
<plugins>
633+
<plugin>
634+
<groupId>net.alchim31.maven</groupId>
635+
<artifactId>scala-maven-plugin</artifactId>
636+
</plugin>
637+
<plugin>
638+
<groupId>org.scalatest</groupId>
639+
<artifactId>scalatest-maven-plugin</artifactId>
640+
<configuration>
641+
<reportsDirectory>${project.build.directory}/unittest-reports</reportsDirectory>
642+
<junitxml>.</junitxml>
643+
<filereports>WDF UnitTestSuite.txt</filereports>
644+
<tagsToExclude>org.apache.spark.sql.connector.kinesis.it.IntegrationTestSuite</tagsToExclude>
645+
</configuration>
646+
<executions>
647+
<execution>
648+
<id>test</id>
649+
<goals>
650+
<goal>test</goal>
651+
</goals>
652+
</execution>
653+
</executions>
654+
</plugin>
655+
</plugins>
656+
</build>
657+
</profile>
658+
623659
</profiles>
624660

625661
<distributionManagement>

src/test/scala/org/apache/spark/sql/connector/kinesis/KinesisOptionsSuite.scala

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -215,6 +215,16 @@ class KinesisOptionsSuite extends KinesisTestBase {
215215
options.stsRoleArn.get shouldBe "assumeRoleTest"
216216
options.stsSessionName.get shouldBe "sessionNameTest"
217217
}
218+
219+
test("sink option uses names as published in README") {
220+
ENDPOINT_URL shouldBe "kinesis.endpointUrl"
221+
REGION shouldBe "kinesis.region"
222+
STREAM_NAME shouldBe "kinesis.streamName"
223+
SINK_FLUSH_WAIT_TIME_MILLIS shouldBe "kinesis.sink.flushWaitTimeMs"
224+
SINK_RECORD_MAX_BUFFERED_TIME shouldBe "kinesis.sink.recordMaxBufferedTimeMs"
225+
SINK_MAX_CONNECTIONS shouldBe "kinesis.sink.maxConnections"
226+
SINK_AGGREGATION_ENABLED shouldBe "kinesis.sink.aggregationEnabled"
227+
}
218228
}
219229

220230
object KinesisOptionsSuite {

0 commit comments

Comments
 (0)