Skip to content

Commit 6082336

Browse files
JonasJebingmarcphilipp
JonasJebing
and
marcphilipp
authored
Add ParameterizedTest#argumentCountValidation (#4045)
This allows parameterized tests to fail when there are more arguments provided than declared by the test method. This is done in a backwards compatible way by only enabling that validation when the new `junit.jupiter.params.argumentCountValidation` configuration parameter is set to `strict` or `ParameterizedTest#argumentCountValidation` is set to `ArgumentCountValidationMode.STRICT`. Resolves #3708. --------- Co-authored-by: Marc Philipp <mail@marcphilipp.de>
1 parent 949239a commit 6082336

File tree

8 files changed

+309
-4
lines changed

8 files changed

+309
-4
lines changed

‎documentation/src/docs/asciidoc/release-notes/release-notes-5.12.0-M1.adoc

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,12 @@ JUnit repository on GitHub.
9595
a test-scoped `ExtensionContext` in `Extension` methods called during test class
9696
instantiation. This behavior will become the default in future versions of JUnit.
9797
* `@TempDir` is now supported on test class constructors.
98+
* Parameterized tests now support argument count validation.
99+
If the `junit.jupiter.params.argumentCountValidation=strict` configuration parameter
100+
or the `@ParameterizedTest(argumentCountValidation = STRICT)` attribute is set, any
101+
mismatch between the declared number of arguments and the number of arguments provided
102+
by the arguments source will result in an error. By default, it's still only an error if
103+
there are fewer arguments provided than declared.
98104
* The new `PreInterruptCallback` extension point defines the API for `Extensions` that
99105
wish to be called prior to invocations of `Thread#interrupt()` by the `@Timeout`
100106
extension.

‎documentation/src/docs/asciidoc/user-guide/writing-tests.adoc

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2020,6 +2020,29 @@ The following annotations are repeatable:
20202020
* `@CsvFileSource`
20212021
* `@ArgumentsSource`
20222022

2023+
[[writing-tests-parameterized-tests-argument-count-validation]]
2024+
==== Argument Count Validation
2025+
2026+
WARNING: Argument count validation is currently an _experimental_ feature. You're invited to
2027+
give it a try and provide feedback to the JUnit team so they can improve and eventually
2028+
<<api-evolution, promote>> this feature.
2029+
2030+
By default, when an arguments source provides more arguments than the test method needs,
2031+
those additional arguments are ignored and the test executes as usual.
2032+
This can lead to bugs where arguments are never passed to the parameterized test method.
2033+
2034+
To prevent this, you can set argument count validation to 'strict'.
2035+
Then, any additional arguments will cause an error instead.
2036+
2037+
To change this behavior for all tests, set the `junit.jupiter.params.argumentCountValidation`
2038+
<<running-tests-config-params, configuration parameter>> to `strict`.
2039+
To change this behavior for a single test,
2040+
use the `argumentCountValidation` attribute of the `@ParameterizedTest` annotation:
2041+
2042+
[source,java,indent=0]
2043+
----
2044+
include::{testDir}/example/ParameterizedTestDemo.java[tags=argument_count_validation]
2045+
----
20232046

20242047
[[writing-tests-parameterized-tests-argument-conversion]]
20252048
==== Argument Conversion

‎documentation/src/test/java/example/ParameterizedTestDemo.java

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@
5151
import org.junit.jupiter.api.extension.ExtensionContext;
5252
import org.junit.jupiter.api.extension.ParameterContext;
5353
import org.junit.jupiter.api.parallel.Execution;
54+
import org.junit.jupiter.params.ArgumentCountValidationMode;
5455
import org.junit.jupiter.params.ParameterizedTest;
5556
import org.junit.jupiter.params.aggregator.AggregateWith;
5657
import org.junit.jupiter.params.aggregator.ArgumentsAccessor;
@@ -607,4 +608,13 @@ static Stream<String> otherProvider() {
607608
return Stream.of("bar");
608609
}
609610
// end::repeatable_annotations[]
611+
612+
@extensions.ExpectToFail
613+
// tag::argument_count_validation[]
614+
@ParameterizedTest(argumentCountValidation = ArgumentCountValidationMode.STRICT)
615+
@CsvSource({ "42, -666" })
616+
void testWithArgumentCountValidation(int number) {
617+
assertTrue(number > 0);
618+
}
619+
// end::argument_count_validation[]
610620
}

‎junit-jupiter-params/src/main/java/org/junit/jupiter/params/ArgumentCountValidationMode.java

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
/*
2+
* Copyright 2015-2024 the original author or authors.
3+
*
4+
* All rights reserved. This program and the accompanying materials are
5+
* made available under the terms of the Eclipse Public License v2.0 which
6+
* accompanies this distribution and is available at
7+
*
8+
* https://www.eclipse.org/legal/epl-v20.html
9+
*/
10+
11+
package org.junit.jupiter.params;
12+
13+
import org.apiguardian.api.API;
14+
import org.junit.jupiter.params.provider.ArgumentsSource;
15+
16+
/**
17+
* Enumeration of argument count validation modes for {@link ParameterizedTest @ParameterizedTest}.
18+
*
19+
* <p>When an {@link ArgumentsSource} provides more arguments than declared by the test method,
20+
* there might be a bug in the test method or the {@link ArgumentsSource}.
21+
* By default, the additional arguments are ignored.
22+
* {@link ArgumentCountValidationMode} allows you to control how additional arguments are handled.
23+
*
24+
* @since 5.12
25+
* @see ParameterizedTest
26+
*/
27+
@API(status = API.Status.EXPERIMENTAL, since = "5.12")
28+
public enum ArgumentCountValidationMode {
29+
/**
30+
* Use the default validation mode.
31+
*
32+
* <p>The default validation mode may be changed via the
33+
* {@value ArgumentCountValidator#ARGUMENT_COUNT_VALIDATION_KEY} configuration parameter
34+
* (see the User Guide for details on configuration parameters).
35+
*/
36+
DEFAULT,
37+
38+
/**
39+
* Use the "none" argument count validation mode.
40+
*
41+
* <p>When there are more arguments provided than declared by the test method,
42+
* these additional arguments are ignored.
43+
*/
44+
NONE,
45+
46+
/**
47+
* Use the strict argument count validation mode.
48+
*
49+
* <p>When there are more arguments provided than declared by the test method, this raises an error.
50+
*/
51+
STRICT,
52+
}

‎junit-jupiter-params/src/main/java/org/junit/jupiter/params/ArgumentCountValidator.java

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
/*
2+
* Copyright 2015-2024 the original author or authors.
3+
*
4+
* All rights reserved. This program and the accompanying materials are
5+
* made available under the terms of the Eclipse Public License v2.0 which
6+
* accompanies this distribution and is available at
7+
*
8+
* https://www.eclipse.org/legal/epl-v20.html
9+
*/
10+
11+
package org.junit.jupiter.params;
12+
13+
import java.lang.reflect.Method;
14+
import java.util.Arrays;
15+
import java.util.Optional;
16+
17+
import org.junit.jupiter.api.extension.ExtensionConfigurationException;
18+
import org.junit.jupiter.api.extension.ExtensionContext;
19+
import org.junit.jupiter.api.extension.InvocationInterceptor;
20+
import org.junit.jupiter.api.extension.ReflectiveInvocationContext;
21+
import org.junit.jupiter.params.provider.Arguments;
22+
import org.junit.platform.commons.logging.Logger;
23+
import org.junit.platform.commons.logging.LoggerFactory;
24+
import org.junit.platform.commons.util.Preconditions;
25+
26+
class ArgumentCountValidator implements InvocationInterceptor {
27+
private static final Logger logger = LoggerFactory.getLogger(ArgumentCountValidator.class);
28+
29+
static final String ARGUMENT_COUNT_VALIDATION_KEY = "junit.jupiter.params.argumentCountValidation";
30+
private static final ExtensionContext.Namespace NAMESPACE = ExtensionContext.Namespace.create(
31+
ArgumentCountValidator.class);
32+
33+
private final ParameterizedTestMethodContext methodContext;
34+
private final Arguments arguments;
35+
36+
ArgumentCountValidator(ParameterizedTestMethodContext methodContext, Arguments arguments) {
37+
this.methodContext = methodContext;
38+
this.arguments = arguments;
39+
}
40+
41+
@Override
42+
public void interceptTestTemplateMethod(InvocationInterceptor.Invocation<Void> invocation,
43+
ReflectiveInvocationContext<Method> invocationContext, ExtensionContext extensionContext) throws Throwable {
44+
validateArgumentCount(extensionContext, arguments);
45+
invocation.proceed();
46+
}
47+
48+
private ExtensionContext.Store getStore(ExtensionContext context) {
49+
return context.getRoot().getStore(NAMESPACE);
50+
}
51+
52+
private void validateArgumentCount(ExtensionContext extensionContext, Arguments arguments) {
53+
ArgumentCountValidationMode argumentCountValidationMode = getArgumentCountValidationMode(extensionContext);
54+
switch (argumentCountValidationMode) {
55+
case DEFAULT:
56+
case NONE:
57+
return;
58+
case STRICT:
59+
int testParamCount = extensionContext.getRequiredTestMethod().getParameterCount();
60+
int argumentsCount = arguments.get().length;
61+
Preconditions.condition(testParamCount == argumentsCount, () -> String.format(
62+
"Configuration error: the @ParameterizedTest has %s argument(s) but there were %s argument(s) provided.%nNote: the provided arguments are %s",
63+
testParamCount, argumentsCount, Arrays.toString(arguments.get())));
64+
break;
65+
default:
66+
throw new ExtensionConfigurationException(
67+
"Unsupported argument count validation mode: " + argumentCountValidationMode);
68+
}
69+
}
70+
71+
private ArgumentCountValidationMode getArgumentCountValidationMode(ExtensionContext extensionContext) {
72+
ParameterizedTest parameterizedTest = methodContext.annotation;
73+
if (parameterizedTest.argumentCountValidation() != ArgumentCountValidationMode.DEFAULT) {
74+
return parameterizedTest.argumentCountValidation();
75+
}
76+
else {
77+
return getArgumentCountValidationModeConfiguration(extensionContext);
78+
}
79+
}
80+
81+
private ArgumentCountValidationMode getArgumentCountValidationModeConfiguration(ExtensionContext extensionContext) {
82+
String key = ARGUMENT_COUNT_VALIDATION_KEY;
83+
ArgumentCountValidationMode fallback = ArgumentCountValidationMode.NONE;
84+
ExtensionContext.Store store = getStore(extensionContext);
85+
return store.getOrComputeIfAbsent(key, __ -> {
86+
Optional<String> optionalConfigValue = extensionContext.getConfigurationParameter(key);
87+
if (optionalConfigValue.isPresent()) {
88+
String configValue = optionalConfigValue.get();
89+
Optional<ArgumentCountValidationMode> enumValue = Arrays.stream(
90+
ArgumentCountValidationMode.values()).filter(
91+
mode -> mode.name().equalsIgnoreCase(configValue)).findFirst();
92+
if (enumValue.isPresent()) {
93+
logger.config(() -> String.format(
94+
"Using ArgumentCountValidationMode '%s' set via the '%s' configuration parameter.",
95+
enumValue.get().name(), key));
96+
return enumValue.get();
97+
}
98+
else {
99+
logger.warn(() -> String.format(
100+
"Invalid ArgumentCountValidationMode '%s' set via the '%s' configuration parameter. "
101+
+ "Falling back to the %s default value.",
102+
configValue, key, fallback.name()));
103+
return fallback;
104+
}
105+
}
106+
else {
107+
return fallback;
108+
}
109+
}, ArgumentCountValidationMode.class);
110+
}
111+
}

‎junit-jupiter-params/src/main/java/org/junit/jupiter/params/ParameterizedTest.java

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.apiguardian.api.API;
2323
import org.junit.jupiter.api.TestTemplate;
2424
import org.junit.jupiter.api.extension.ExtendWith;
25+
import org.junit.jupiter.params.provider.ArgumentsSource;
2526

2627
/**
2728
* {@code @ParameterizedTest} is used to signal that the annotated method is a
@@ -305,4 +306,21 @@
305306
@API(status = EXPERIMENTAL, since = "5.12")
306307
boolean requireArguments() default true;
307308

309+
/**
310+
* Configure how the number of arguments provided by an {@link ArgumentsSource} are validated.
311+
*
312+
* <p>Defaults to {@link ArgumentCountValidationMode#DEFAULT}.
313+
*
314+
* <p>When an {@link ArgumentsSource} provides more arguments than declared by the test method,
315+
* there might be a bug in the test method or the {@link ArgumentsSource}.
316+
* By default, the additional arguments are ignored.
317+
* {@code argumentCountValidation} allows you to control how additional arguments are handled.
318+
* The default can be configured via the {@value ArgumentCountValidator#ARGUMENT_COUNT_VALIDATION_KEY}
319+
* configuration parameter (see the User Guide for details on configuration parameters).
320+
*
321+
* @since 5.12
322+
* @see ArgumentCountValidationMode
323+
*/
324+
@API(status = EXPERIMENTAL, since = "5.12")
325+
ArgumentCountValidationMode argumentCountValidation() default ArgumentCountValidationMode.DEFAULT;
308326
}

‎junit-jupiter-params/src/main/java/org/junit/jupiter/params/ParameterizedTestInvocationContext.java

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,6 @@
1010

1111
package org.junit.jupiter.params;
1212

13-
import static java.util.Collections.singletonList;
14-
1513
import java.util.Arrays;
1614
import java.util.List;
1715

@@ -47,8 +45,9 @@ public String getDisplayName(int invocationIndex) {
4745

4846
@Override
4947
public List<Extension> getAdditionalExtensions() {
50-
return singletonList(
51-
new ParameterizedTestParameterResolver(this.methodContext, this.consumedArguments, this.invocationIndex));
48+
return Arrays.asList(
49+
new ParameterizedTestParameterResolver(this.methodContext, this.consumedArguments, this.invocationIndex),
50+
new ArgumentCountValidator(this.methodContext, this.arguments));
5251
}
5352

5453
private static Object[] consumedArguments(ParameterizedTestMethodContext methodContext, Object[] arguments) {

‎jupiter-tests/src/test/java/org/junit/jupiter/params/ParameterizedTestIntegrationTests.java

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,7 @@
116116
import org.junit.platform.testkit.engine.EngineExecutionResults;
117117
import org.junit.platform.testkit.engine.EngineTestKit;
118118
import org.junit.platform.testkit.engine.Event;
119+
import org.junit.platform.testkit.engine.EventConditions;
119120
import org.opentest4j.TestAbortedException;
120121

121122
/**
@@ -1112,6 +1113,74 @@ private EngineExecutionResults execute(String methodName, Class<?>... methodPara
11121113

11131114
}
11141115

1116+
@Nested
1117+
class UnusedArgumentsWithStrictArgumentsCountIntegrationTests {
1118+
@Test
1119+
void failsWithArgumentsSourceProvidingUnusedArguments() {
1120+
var results = execute(ArgumentCountValidationMode.STRICT, UnusedArgumentsTestCase.class,
1121+
"testWithTwoUnusedStringArgumentsProvider", String.class);
1122+
results.allEvents().assertThatEvents() //
1123+
.haveExactly(1, event(EventConditions.finishedWithFailure(message(String.format(
1124+
"Configuration error: the @ParameterizedTest has 1 argument(s) but there were 2 argument(s) provided.%nNote: the provided arguments are [foo, unused1]")))));
1125+
}
1126+
1127+
@Test
1128+
void failsWithMethodSourceProvidingUnusedArguments() {
1129+
var results = execute(ArgumentCountValidationMode.STRICT, UnusedArgumentsTestCase.class,
1130+
"testWithMethodSourceProvidingUnusedArguments", String.class);
1131+
results.allEvents().assertThatEvents() //
1132+
.haveExactly(1, event(EventConditions.finishedWithFailure(message(String.format(
1133+
"Configuration error: the @ParameterizedTest has 1 argument(s) but there were 2 argument(s) provided.%nNote: the provided arguments are [foo, unused1]")))));
1134+
}
1135+
1136+
@Test
1137+
void failsWithCsvSourceUnusedArgumentsAndStrictArgumentCountValidationAnnotationAttribute() {
1138+
var results = execute(ArgumentCountValidationMode.NONE, UnusedArgumentsTestCase.class,
1139+
"testWithStrictArgumentCountValidation", String.class);
1140+
results.allEvents().assertThatEvents() //
1141+
.haveExactly(1, event(EventConditions.finishedWithFailure(message(String.format(
1142+
"Configuration error: the @ParameterizedTest has 1 argument(s) but there were 2 argument(s) provided.%nNote: the provided arguments are [foo, unused1]")))));
1143+
}
1144+
1145+
@Test
1146+
void failsWithCsvSourceUnusedArgumentsButExecutesRemainingArgumentsWhereThereIsNoUnusedArgument() {
1147+
var results = execute(ArgumentCountValidationMode.STRICT, UnusedArgumentsTestCase.class,
1148+
"testWithCsvSourceContainingDifferentNumbersOfArguments", String.class);
1149+
results.allEvents().assertThatEvents() //
1150+
.haveExactly(1, event(EventConditions.finishedWithFailure(message(String.format(
1151+
"Configuration error: the @ParameterizedTest has 1 argument(s) but there were 2 argument(s) provided.%nNote: the provided arguments are [foo, unused1]"))))) //
1152+
.haveExactly(1,
1153+
event(test(), displayName("[2] argument=bar"), finishedWithFailure(message("bar"))));
1154+
}
1155+
1156+
@Test
1157+
void executesWithCsvSourceUnusedArgumentsAndArgumentCountValidationAnnotationAttribute() {
1158+
var results = execute(ArgumentCountValidationMode.NONE, UnusedArgumentsTestCase.class,
1159+
"testWithNoneArgumentCountValidation", String.class);
1160+
results.allEvents().assertThatEvents() //
1161+
.haveExactly(1,
1162+
event(test(), displayName("[1] argument=foo"), finishedWithFailure(message("foo"))));
1163+
}
1164+
1165+
@Test
1166+
void executesWithMethodSourceProvidingUnusedArguments() {
1167+
var results = execute(ArgumentCountValidationMode.STRICT, RepeatableSourcesTestCase.class,
1168+
"testWithRepeatableCsvSource", String.class);
1169+
results.allEvents().assertThatEvents() //
1170+
.haveExactly(1, event(test(), displayName("[1] argument=a"), finishedWithFailure(message("a")))) //
1171+
.haveExactly(1, event(test(), displayName("[2] argument=b"), finishedWithFailure(message("b"))));
1172+
}
1173+
1174+
private EngineExecutionResults execute(ArgumentCountValidationMode configurationValue, Class<?> javaClass,
1175+
String methodName, Class<?>... methodParameterTypes) {
1176+
return EngineTestKit.engine(new JupiterTestEngine()) //
1177+
.selectors(selectMethod(javaClass, methodName, methodParameterTypes)) //
1178+
.configurationParameter(ArgumentCountValidator.ARGUMENT_COUNT_VALIDATION_KEY,
1179+
configurationValue.name().toLowerCase()) //
1180+
.execute();
1181+
}
1182+
}
1183+
11151184
@Nested
11161185
class RepeatableSourcesIntegrationTests {
11171186

@@ -2028,6 +2097,23 @@ void testWithFieldSourceProvidingUnusedArguments(String argument) {
20282097
static Supplier<Stream<Arguments>> unusedArgumentsProviderField = //
20292098
() -> Stream.of(arguments("foo", "unused1"), arguments("bar", "unused2"));
20302099

2100+
@ParameterizedTest(argumentCountValidation = ArgumentCountValidationMode.STRICT)
2101+
@CsvSource({ "foo, unused1" })
2102+
void testWithStrictArgumentCountValidation(String argument) {
2103+
fail(argument);
2104+
}
2105+
2106+
@ParameterizedTest(argumentCountValidation = ArgumentCountValidationMode.NONE)
2107+
@CsvSource({ "foo, unused1" })
2108+
void testWithNoneArgumentCountValidation(String argument) {
2109+
fail(argument);
2110+
}
2111+
2112+
@ParameterizedTest
2113+
@CsvSource({ "foo, unused1", "bar" })
2114+
void testWithCsvSourceContainingDifferentNumbersOfArguments(String argument) {
2115+
fail(argument);
2116+
}
20312117
}
20322118

20332119
static class LifecycleTestCase {

0 commit comments

Comments
 (0)