Skip to content

Don't run failed unit tests by default #5248

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Mar 4, 2020
Merged

Don't run failed unit tests by default #5248

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
da7a1fd
Add a define for XFAIL
Feb 27, 2020
186855f
Enable two tests that pass
Feb 27, 2020
a2c45e8
Replace mayfail with XFAIL
Feb 27, 2020
9fcd171
Running expected failure tests after running regular unit tests
Feb 27, 2020
44d1ee4
Include hidden tests when counting tests
Feb 27, 2020
18cca67
Add running XFAIL tests to other jobs
Feb 27, 2020
6176871
Add and invoke a ctest target for checking expected failure unit tests
Mar 3, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions .travis.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -243,7 +243,7 @@ jobs:
- cmake -S . -Bbuild '-DCMAKE_BUILD_TYPE=Release' '-DCMAKE_CXX_COMPILER=/usr/bin/g++-7' '-DCMAKE_USE_CUDD=true' -DCMAKE_CXX_FLAGS="-DBDD_GUARDS" '-DWITH_MEMORY_ANALYZER=On'
- git submodule update --init --recursive
- cmake --build build -- -j4
script: (cd build; ctest -V -L CORE -j2)
script: (cd build; ctest -V -L CORE -j2; ctest -V -R unit-xfail -j2)

# cmake build using clang++-6
- stage: Test different OS/CXX/Flags
Expand Down Expand Up @@ -280,7 +280,7 @@ jobs:
- cmake -S . -Bbuild '-DCMAKE_BUILD_TYPE=Release' '-DCMAKE_CXX_COMPILER=/usr/bin/clang++-7' '-DCMAKE_CXX_FLAGS=-Qunused-arguments' '-DWITH_MEMORY_ANALYZER=On'
- git submodule update --init --recursive
- cmake --build build -- -j4
script: (cd build; ctest -V -L CORE -j2)
script: (cd build; ctest -V -L CORE -j2; ctest -V -R unit-xfail -j2)

# cmake build on OSX, using default clang
- stage: Test different OS/CXX/Flags
Expand All @@ -300,7 +300,7 @@ jobs:
- cmake -S . -Bbuild '-DCMAKE_BUILD_TYPE=Release' '-DCMAKE_OSX_ARCHITECTURES=x86_64'
- git submodule update --init --recursive
- cmake --build build -- -j4
script: (cd build; ctest -V -L CORE -j2)
script: (cd build; ctest -V -L CORE -j2; ctest -V -R unit-xfail -j2)


# Run Coverity
Expand Down Expand Up @@ -355,9 +355,13 @@ script:
- env PATH=$PATH:`pwd`/src/solvers UBSAN_OPTIONS=print_stacktrace=1 make -C regression/cbmc test-cprover-smt2
- make -C unit "CXX=${COMPILER} ${EXTRA_CXXFLAGS}" -j2
- make -C unit test
- echo "Running expected failure tests"
- make TAGS="[!shouldfail]" -C unit test
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What about CMake configurations? Does anything need to be done for those?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good spot, fixed

- env UBSAN_OPTIONS=print_stacktrace=1 make -C jbmc/regression test-parallel "CXX=${COMPILER} ${EXTRA_CXXFLAGS}" -j2 JOBS=2
- make -C jbmc/unit "CXX=${COMPILER} ${EXTRA_CXXFLAGS}" -j2
- make -C jbmc/unit test
- echo "Running expected failure tests"
- make TAGS="[!shouldfail]" -C jbmc/unit test

before_cache:
- ccache -s
Expand Down
4 changes: 4 additions & 0 deletions buildspec-linux-clang.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,14 @@ phases:
post_build:
commands:
- make -C unit test
- echo "Running expected failure tests"
- make TAGS="[!shouldfail]" -C unit test
- make -C regression test CXX='ccache /usr/bin/clang++-8' CXX_FLAGS='-Qunused-arguments'
- make -C regression/cbmc test-paths-lifo
- env PATH=$PATH:`pwd`/src/solvers make -C regression/cbmc test-cprover-smt2
- make -C jbmc/unit test
- echo "Running expected failure tests"
- make TAGS="[!shouldfail]" -C jbmc/unit test
- make -C jbmc/regression test
- echo Build completed on `date`
cache:
Expand Down
1 change: 1 addition & 0 deletions buildspec-linux-cmake-gcc.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ phases:
post_build:
commands:
- cd build; ctest -V -L CORE -j2
- cd build; ctest -V -R unit-xfail -j2
- echo Build completed on `date`
cache:
paths:
Expand Down
8 changes: 8 additions & 0 deletions buildspec-windows.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,10 @@ phases:
$env:Path = "C:\tools\cygwin\bin;$env:Path"
cmd /c 'call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x64 && bash -c "make -C unit test BUILD_ENV=MSVC" '

- |
$env:Path = "C:\tools\cygwin\bin;$env:Path"
cmd /c 'call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x64 && bash -c "make TAGS=[!shouldfail] -C unit test BUILD_ENV=MSVC" '

- |
$env:Path = "C:\tools\cygwin\bin;$env:Path"
cmd /c 'call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x64 && bash -c "make -C jbmc/regression test BUILD_ENV=MSVC" '
Expand All @@ -65,6 +69,10 @@ phases:
$env:Path = "C:\tools\cygwin\bin;$env:Path"
cmd /c 'call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x64 && bash -c "make -C jbmc/unit test BUILD_ENV=MSVC" '

- |
$env:Path = "C:\tools\cygwin\bin;$env:Path"
cmd /c 'call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x64 && bash -c "make TAGS=[!shouldfail] -C jbmc/unit test BUILD_ENV=MSVC" '

cache:
paths:
- 'c:\clcache\**\*'
4 changes: 4 additions & 0 deletions buildspec.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,14 @@ phases:
post_build:
commands:
- make -C unit test
- echo "Running expected failure tests"
- make TAGS="[!shouldfail]" -C unit test
- make -C regression test
- make -C regression/cbmc test-paths-lifo
- env PATH=$PATH:`pwd`/src/solvers make -C regression/cbmc test-cprover-smt2
- make -C jbmc/unit test
- echo "Running expected failure tests"
- make TAGS="[!shouldfail]" -C jbmc/unit test
- make -C jbmc/regression test
- echo Build completed on `date`
cache:
Expand Down
7 changes: 7 additions & 0 deletions jbmc/unit/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,4 +36,11 @@ add_test(
COMMAND $<TARGET_FILE:java-unit>
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
)

add_test(
NAME java-unit-xfail
COMMAND $<TARGET_FILE:java-unit> "[!shouldfail]"
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
)

set_tests_properties(java-unit PROPERTIES LABELS "CORE;CBMC")
7 changes: 4 additions & 3 deletions jbmc/unit/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,9 +144,10 @@ all: $(CATCH_TEST)
clean: java-testing-utils-clean

test: $(CATCH_TEST)
if ! ./$(CATCH_TEST) -l | grep -q "^$(N_CATCH_TESTS) test cases" ; then \
./$(CATCH_TEST) -l ; fi
./$(CATCH_TEST)
# Include hidden tests by specifying "*,[.]" for tests to count
if ! ./$(CATCH_TEST) "*,[.]" -l | grep -q "^$(N_CATCH_TESTS) matching test cases" ; then \
./$(CATCH_TEST) "*,[.]" -l ; fi
./$(CATCH_TEST) ${TAGS}


###############################################################################
Expand Down
4 changes: 2 additions & 2 deletions jbmc/unit/java_bytecode/ci_lazy_methods/lazy_load_lambdas.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ Author: Diffblue Limited.

SCENARIO(
"Lazy load lambda methods",
"[core][java_bytecode][ci_lazy_methods][lambdas][!mayfail]")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

mayfail doesn’t stop a test from running, it just means test failure isn’t counted.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A test that can pass or fail is bad - since no body will observe if it changes state. I believe this has been erroneously marked as mayfail, since it passes.

"[core][java_bytecode][ci_lazy_methods][lambdas]")
{
GIVEN("A class with some locally declared lambdas")
{
Expand Down Expand Up @@ -192,7 +192,7 @@ SCENARIO(

SCENARIO(
"Lazy load lambda methods in seperate class",
"[core][java_bytecode][ci_lazy_methods][lambdas][!mayfail]")
"[core][java_bytecode][ci_lazy_methods][lambdas]")
{
const symbol_tablet symbol_table = load_java_class_lazy(
"ExternalLambdaAccessor",
Expand Down
6 changes: 3 additions & 3 deletions jbmc/unit/java_bytecode/java_bytecode_convert_method/convert_invoke_dynamic.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,7 @@ void validate_local_variable_lambda_assignment(
SCENARIO(
"Converting invokedynamic with a local lambda",
"[core]"
"[lambdas][java_bytecode][java_bytecode_convert_method][!mayfail]")
"[lambdas][java_bytecode][java_bytecode_convert_method]" XFAIL)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this should be [!shouldfail] (it’s shouldfail in the macro), and I still dont think this should be a macro.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in later commit

{
// NOLINTNEXTLINE(whitespace/braces)
run_test_with_compilers([](const std::string &compiler) {
Expand Down Expand Up @@ -380,7 +380,7 @@ void validate_member_variable_lambda_assignment(
SCENARIO(
"Converting invokedynamic with a member lambda",
"[core]"
"[lamdba][java_bytecode][java_bytecode_convert_method][!mayfail]")
"[lamdba][java_bytecode][java_bytecode_convert_method]" XFAIL)
{
// NOLINTNEXTLINE(whitespace/braces)
run_test_with_compilers([](const std::string &compiler) {
Expand Down Expand Up @@ -548,7 +548,7 @@ void validate_static_member_variable_lambda_assignment(
SCENARIO(
"Converting invokedynamic with a static member lambda",
"[core]"
"[lamdba][java_bytecode][java_bytecode_convert_method][!mayfail]")
"[lamdba][java_bytecode][java_bytecode_convert_method]" XFAIL)
{
// NOLINTNEXTLINE(whitespace/braces)
run_test_with_compilers([](const std::string &compiler) {
Expand Down
3 changes: 1 addition & 2 deletions jbmc/unit/solvers/strings/string_constraint_instantiation/instantiate_not_contains.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,10 +172,9 @@ decision_proceduret::resultt check_sat(const exprt &expr, const namespacet &ns)
return solver();
}

// The [!mayfail] tag allows tests to fail while reporting the failure
SCENARIO(
"instantiate_not_contains",
"[!mayfail][core][solvers][refinement][string_constraint_instantiation]")
"[core][solvers][refinement][string_constraint_instantiation]" XFAIL)
{
// For printing expression
register_language(new_java_bytecode_language);
Expand Down
7 changes: 7 additions & 0 deletions unit/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,4 +65,11 @@ add_test(
COMMAND $<TARGET_FILE:unit>
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
)

add_test(
NAME unit-xfail
COMMAND $<TARGET_FILE:unit> "[!shouldfail]"
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
)

set_tests_properties(unit PROPERTIES LABELS "CORE;CBMC")
7 changes: 4 additions & 3 deletions unit/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -204,9 +204,10 @@ all: $(CATCH_TEST)
clean: testing-utils-clean

test: $(CATCH_TEST)
if ! ./$(CATCH_TEST) -l | grep -q "^$(N_CATCH_TESTS) test cases" ; then \
./$(CATCH_TEST) -l ; fi
./$(CATCH_TEST)
# Include hidden tests by specifying "*,[.]" for tests to count
if ! ./$(CATCH_TEST) "*,[.]" -l | grep -q "^$(N_CATCH_TESTS) matching test cases" ; then \
./$(CATCH_TEST) "*,[.]" -l ; fi
./$(CATCH_TEST) ${TAGS}


###############################################################################
Expand Down
3 changes: 3 additions & 0 deletions unit/testing-utils/use_catch.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,4 +36,7 @@ Author: Michael Tautschnig
#include <util/pragma_pop.def>
#endif

/// Add to the end of test tags to mark a test that is expected to fail
#define XFAIL "[.][!shouldfail]"

#endif // CPROVER_TESTING_UTILS_USE_CATCH_H
4 changes: 2 additions & 2 deletions unit/util/irep_sharing.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

#ifdef SHARING

SCENARIO("irept_sharing_trade_offs", "[!mayfail][core][utils][irept]")
SCENARIO("irept_sharing_trade_offs", "[core][utils][irept]" XFAIL)
{
GIVEN("An irept created with move_to_sub")
{
Expand Down Expand Up @@ -116,7 +116,7 @@ SCENARIO("irept_sharing", "[core][utils][irept]")

#include <util/expr.h>

SCENARIO("exprt_sharing_trade_offs", "[!mayfail][core][utils][exprt]")
SCENARIO("exprt_sharing_trade_offs", "[core][utils][exprt]" XFAIL)
{
GIVEN("An expression created with add_to_operands(std::move(...))")
{
Expand Down