From fe1993e609df405456f8a7fb78f0bbb733f741fd Mon Sep 17 00:00:00 2001 From: Sam Brannen Date: Fri, 14 Dec 2018 14:16:03 +0100 Subject: [PATCH] Document how to verify ALL events with the EngineTestKit in User Guide Issue: #1621 --- .../src/docs/asciidoc/user-guide/testkit.adoc | 48 +++++++++++++- .../test/java/example/ExampleTestCase.java | 10 ++- .../testkit/EngineTestKitAllEventsDemo.java | 63 +++++++++++++++++++ 3 files changed, 118 insertions(+), 3 deletions(-) create mode 100644 documentation/src/test/java/example/testkit/EngineTestKitAllEventsDemo.java diff --git a/documentation/src/docs/asciidoc/user-guide/testkit.adoc b/documentation/src/docs/asciidoc/user-guide/testkit.adoc index 4910d48d560c..ed00091838aa 100644 --- a/documentation/src/docs/asciidoc/user-guide/testkit.adoc +++ b/documentation/src/docs/asciidoc/user-guide/testkit.adoc @@ -72,7 +72,7 @@ For example, if you want to verify the reason that the `skippedTest()` method in <> was skipped, you can do that as follows. -[NOTE] +[TIP] ==== The `assertThatEvents()` method in the following example is a shortcut for `org.assertj.core.api.Assertions.assertThat(events.list())` from the {AssertJ} assertion @@ -99,7 +99,7 @@ include::{testDir}/example/testkit/EngineTestKitSkippedMethodDemo.java[tags=user If you want to verify the type of exception thrown from the `failingTest()` method in <>, you can do that as follows. -[NOTE] +[TIP] ==== For details on what _conditions_ are available for use with AssertJ assertions against events and execution results, consult the Javadoc for `{EventConditions}` and @@ -117,3 +117,47 @@ include::{testDir}/example/testkit/EngineTestKitFailedMethodDemo.java[tags=user_ <5> Assert that the recorded _test_ events contain exactly one failing test named `failingTest` with an exception of type `ArithmeticException` and an error message equal to `"/ by zero"`. + +Although typically unnecessary, there are times when you need to verify **all** of the +events fired during the execution of a `TestPlan`. The following test demonstrates how to +achieve this via the `assertEventsMatchExactly()` method in the `EngineTestKit` API. + +[TIP] +==== +Since `assertEventsMatchExactly()` matches conditions exactly in the order in which the +events were fired, <> has been +annotated with `@TestMethodOrder(OrderAnnotation.class)` and each test method has been +annotated with `@Order(...)`. This allows us to enforce the order in which the test +methods are executed, which in turn allows our `verifyAllJupiterEvents()` test to be +reliable. +==== + +[source,java,indent=0] +---- +include::{testDir}/example/testkit/EngineTestKitAllEventsDemo.java[tags=user_guide] +---- +<1> Select the JUnit Jupiter `TestEngine`. +<2> Select the <> test class. +<3> Execute the `TestPlan`. +<4> Filter by _all_ events. +<5> Print all events to the standard output stream (`STD_OUT`) for debugging purposes. +<6> Assert _all_ events in exactly the order in which they were fired by the test engine. + +The `debug()` invocation from the preceding example results in output similar to the +following. + +[source,options="nowrap"] +---- +All Events: + Event [type = STARTED, testDescriptor = JupiterEngineDescriptor: [engine:junit-jupiter], timestamp = 2018-12-14T12:45:14.082280Z, payload = null] + Event [type = STARTED, testDescriptor = ClassTestDescriptor: [engine:junit-jupiter]/[class:example.ExampleTestCase], timestamp = 2018-12-14T12:45:14.089339Z, payload = null] + Event [type = SKIPPED, testDescriptor = TestMethodTestDescriptor: [engine:junit-jupiter]/[class:example.ExampleTestCase]/[method:skippedTest()], timestamp = 2018-12-14T12:45:14.094314Z, payload = 'for demonstration purposes'] + Event [type = STARTED, testDescriptor = TestMethodTestDescriptor: [engine:junit-jupiter]/[class:example.ExampleTestCase]/[method:succeedingTest()], timestamp = 2018-12-14T12:45:14.095182Z, payload = null] + Event [type = FINISHED, testDescriptor = TestMethodTestDescriptor: [engine:junit-jupiter]/[class:example.ExampleTestCase]/[method:succeedingTest()], timestamp = 2018-12-14T12:45:14.104922Z, payload = TestExecutionResult [status = SUCCESSFUL, throwable = null]] + Event [type = STARTED, testDescriptor = TestMethodTestDescriptor: [engine:junit-jupiter]/[class:example.ExampleTestCase]/[method:abortedTest()], timestamp = 2018-12-14T12:45:14.106121Z, payload = null] + Event [type = FINISHED, testDescriptor = TestMethodTestDescriptor: [engine:junit-jupiter]/[class:example.ExampleTestCase]/[method:abortedTest()], timestamp = 2018-12-14T12:45:14.109956Z, payload = TestExecutionResult [status = ABORTED, throwable = org.opentest4j.TestAbortedException: Assumption failed: abc does not contain Z]] + Event [type = STARTED, testDescriptor = TestMethodTestDescriptor: [engine:junit-jupiter]/[class:example.ExampleTestCase]/[method:failingTest()], timestamp = 2018-12-14T12:45:14.110680Z, payload = null] + Event [type = FINISHED, testDescriptor = TestMethodTestDescriptor: [engine:junit-jupiter]/[class:example.ExampleTestCase]/[method:failingTest()], timestamp = 2018-12-14T12:45:14.111217Z, payload = TestExecutionResult [status = FAILED, throwable = java.lang.ArithmeticException: / by zero]] + Event [type = FINISHED, testDescriptor = ClassTestDescriptor: [engine:junit-jupiter]/[class:example.ExampleTestCase], timestamp = 2018-12-14T12:45:14.113731Z, payload = TestExecutionResult [status = SUCCESSFUL, throwable = null]] + Event [type = FINISHED, testDescriptor = JupiterEngineDescriptor: [engine:junit-jupiter], timestamp = 2018-12-14T12:45:14.113806Z, payload = TestExecutionResult [status = SUCCESSFUL, throwable = null]] +---- diff --git a/documentation/src/test/java/example/ExampleTestCase.java b/documentation/src/test/java/example/ExampleTestCase.java index 551d2805ddb4..d22c3c810d0a 100644 --- a/documentation/src/test/java/example/ExampleTestCase.java +++ b/documentation/src/test/java/example/ExampleTestCase.java @@ -18,30 +18,38 @@ import example.util.Calculator; import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.MethodOrderer.OrderAnnotation; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; +@TestMethodOrder(OrderAnnotation.class) public class ExampleTestCase { private final Calculator calculator = new Calculator(); @Test @Disabled("for demonstration purposes") + @Order(1) void skippedTest() { // skipped ... } @Test + @Order(2) void succeedingTest() { assertEquals(42, calculator.multiply(6, 7)); } @Test + @Order(3) void abortedTest() { - assumeTrue("abc".contains("Z")); + assumeTrue("abc".contains("Z"), "abc does not contain Z"); // aborted ... } @Test + @Order(4) void failingTest() { // The following throws an ArithmeticException: "/ by zero" calculator.divide(1, 0); diff --git a/documentation/src/test/java/example/testkit/EngineTestKitAllEventsDemo.java b/documentation/src/test/java/example/testkit/EngineTestKitAllEventsDemo.java new file mode 100644 index 000000000000..f6bfac57d8b8 --- /dev/null +++ b/documentation/src/test/java/example/testkit/EngineTestKitAllEventsDemo.java @@ -0,0 +1,63 @@ +/* + * Copyright 2015-2018 the original author or authors. + * + * All rights reserved. This program and the accompanying materials are + * made available under the terms of the Eclipse Public License v2.0 which + * accompanies this distribution and is available at + * + * http://www.eclipse.org/legal/epl-v20.html + */ + +package example.testkit; + +// @formatter:off +// tag::user_guide[] + +import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass; +import static org.junit.platform.testkit.engine.EventConditions.abortedWithReason; +import static org.junit.platform.testkit.engine.EventConditions.container; +import static org.junit.platform.testkit.engine.EventConditions.engine; +import static org.junit.platform.testkit.engine.EventConditions.event; +import static org.junit.platform.testkit.engine.EventConditions.finishedSuccessfully; +import static org.junit.platform.testkit.engine.EventConditions.finishedWithFailure; +import static org.junit.platform.testkit.engine.EventConditions.skippedWithReason; +import static org.junit.platform.testkit.engine.EventConditions.started; +import static org.junit.platform.testkit.engine.EventConditions.test; +import static org.junit.platform.testkit.engine.TestExecutionResultConditions.isA; +import static org.junit.platform.testkit.engine.TestExecutionResultConditions.message; + +import example.ExampleTestCase; + +import org.junit.jupiter.api.Test; +import org.junit.platform.testkit.engine.EngineTestKit; +import org.opentest4j.TestAbortedException; + +class EngineTestKitAllEventsDemo { + + @Test + void verifyAllJupiterEvents() { + EngineTestKit.engine("junit-jupiter") // <1> + .selectors(selectClass(ExampleTestCase.class)) // <2> + .execute() // <3> + .all() // <4> + .debug() // <5> + .assertEventsMatchExactly( // <6> + event(engine(), started()), + event(container(ExampleTestCase.class), started()), + event(test("skippedTest"), skippedWithReason("for demonstration purposes")), + event(test("succeedingTest"), started()), + event(test("succeedingTest"), finishedSuccessfully()), + event(test("abortedTest"), started()), + event(test("abortedTest"), + abortedWithReason(isA(TestAbortedException.class), + message(m -> m.contains("abc does not contain Z")))), + event(test("failingTest"), started()), + event(test("failingTest"), + finishedWithFailure(isA(ArithmeticException.class), message("/ by zero"))), + event(container(ExampleTestCase.class), finishedSuccessfully()), + event(engine(), finishedSuccessfully())); + } + +} +// end::user_guide[] +// @formatter:on