Skip to content

Commit ca3d5d4

Browse files
author
Andrew Or
committed
Limit kinesis test dependencies
KinesisReceiverSuite really doesn't need to depend on TestSuiteBase. The issue is that TestSuiteBase transitively depends on Spark core tests (SparkFunSuite), but there is no simple way in maven to express transitive test dependencies. To simplify the build structure of this module, we should just have KinesisReceiverSuite extend directly from FunSuite.
1 parent f24e09c commit ca3d5d4

File tree

1 file changed

+12
-8
lines changed

1 file changed

+12
-8
lines changed

extras/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisReceiverSuite.scala

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -26,18 +26,23 @@ import com.amazonaws.services.kinesis.clientlibrary.lib.worker.InitialPositionIn
2626
import com.amazonaws.services.kinesis.clientlibrary.types.ShutdownReason
2727
import com.amazonaws.services.kinesis.model.Record
2828
import org.mockito.Mockito._
29-
import org.scalatest.{BeforeAndAfter, Matchers}
29+
// scalastyle:off
30+
// To avoid introducing a dependency on Spark core tests, simply use scalatest's FunSuite
31+
// here instead of our own SparkFunSuite. Introducing the dependency has caused problems
32+
// in the past (SPARK-8781) that are complicated by bugs in the maven shade plugin (MSHADE-148).
33+
import org.scalatest.{BeforeAndAfter, FunSuite, Matchers}
3034
import org.scalatest.mock.MockitoSugar
3135

3236
import org.apache.spark.storage.StorageLevel
33-
import org.apache.spark.streaming.{Milliseconds, Seconds, StreamingContext, TestSuiteBase}
37+
import org.apache.spark.streaming.{Milliseconds, Seconds, StreamingContext}
3438
import org.apache.spark.util.{Clock, ManualClock, Utils}
3539

3640
/**
3741
* Suite of Kinesis streaming receiver tests focusing mostly on the KinesisRecordProcessor
3842
*/
39-
class KinesisReceiverSuite extends TestSuiteBase with Matchers with BeforeAndAfter
40-
with MockitoSugar {
43+
class KinesisReceiverSuite extends FunSuite with Matchers with BeforeAndAfter
44+
with MockitoSugar {
45+
// scalastyle:on
4146

4247
val app = "TestKinesisReceiver"
4348
val stream = "mySparkStream"
@@ -57,24 +62,23 @@ class KinesisReceiverSuite extends TestSuiteBase with Matchers with BeforeAndAft
5762
var checkpointStateMock: KinesisCheckpointState = _
5863
var currentClockMock: Clock = _
5964

60-
override def beforeFunction(): Unit = {
65+
before {
6166
receiverMock = mock[KinesisReceiver]
6267
checkpointerMock = mock[IRecordProcessorCheckpointer]
6368
checkpointClockMock = mock[ManualClock]
6469
checkpointStateMock = mock[KinesisCheckpointState]
6570
currentClockMock = mock[Clock]
6671
}
6772

68-
override def afterFunction(): Unit = {
69-
super.afterFunction()
73+
after {
7074
// Since this suite was originally written using EasyMock, add this to preserve the old
7175
// mocking semantics (see SPARK-5735 for more details)
7276
verifyNoMoreInteractions(receiverMock, checkpointerMock, checkpointClockMock,
7377
checkpointStateMock, currentClockMock)
7478
}
7579

7680
test("KinesisUtils API") {
77-
val ssc = new StreamingContext(master, framework, batchDuration)
81+
val ssc = new StreamingContext("local[2]", getClass.getSimpleName, Seconds(1))
7882
// Tests the API, does not actually test data receiving
7983
val kinesisStream1 = KinesisUtils.createStream(ssc, "mySparkStream",
8084
"https://kinesis.us-west-2.amazonaws.com", Seconds(2),

0 commit comments

Comments
 (0)