testrunner: made -x work with individual tests (danmar#8362)

Author: firewave

.github/workflows/CI-unixish.yml

@@ -679,6 +679,13 @@ jobs:
           test -z "$(nm processexecutor.o)"
           # TODO: test NO_* defines
 
+      - name: Test testrunner inclusion/exclusion
+        run: |
+          ! ./testrunner -d TestUtils | grep -v TestUtils > /dev/null
+          ! ./testrunner -d TestUtils::trim | grep -v TestUtils::trim > /dev/null
+          ! ./testrunner -d -x TestUtils | grep TestUtils > /dev/null
+          ! ./testrunner -d -x TestUtils:trim | grep TestUtils:trim > /dev/null
+
       - name: Show all ignored files
         if: false  # TODO: currently lists all the contents of ignored folders - we only need what actually matched
         run: |

test/fixture.cpp

@@ -97,19 +97,22 @@ bool TestFixture::prepareTest(const char testname[])
     prepareTestInternal();
 
     // Check if tests should be executed
-    if (testToRun.empty() || testToRun == testname) {
-        // Tests will be executed - prepare them
-        mTestname = testname;
-        ++countTests;
-        if (quiet_tests) {
-            std::putchar('.'); // Use putchar to write through redirection of std::cout/cerr
-            std::fflush(stdout);
-        } else {
-            std::cout << classname << "::" << mTestname << std::endl;
-        }
-        return !dry_run;
+    if (!testsToRun.empty()) {
+        const bool match = testsToRun.count(testname);
+        if ((match && exclude_tests) || (!match && !exclude_tests))
+            return false;
+    }
+
+    // Tests will be executed - prepare them
+    mTestname = testname;
+    ++countTests;
+    if (quiet_tests) {
+        std::putchar('.'); // Use putchar to write through redirection of std::cout/cerr
+        std::fflush(stdout);
+    } else {
+        std::cout << classname << "::" << mTestname << std::endl;
     }
-    return false;
+    return !dry_run;
 }
 
 void TestFixture::teardownTest()
@@ -350,9 +353,9 @@ void TestFixture::printHelp()
         "    -x                   Exclude the specified tests.\n";
 }
 
-void TestFixture::run(const std::string &str)
+void TestFixture::run(const std::set<std::string> &tests)
 {
-    testToRun = str;
+    testsToRun = tests;
     try {
         if (quiet_tests) {
             std::cout << '\n' << classname << ':';
@@ -380,6 +383,7 @@ void TestFixture::processOptions(const options& args)
 {
     quiet_tests = args.quiet();
     dry_run = args.dry_run();
+    exclude_tests = args.exclude_tests();
     exename = args.exe();
 }
 
@@ -388,27 +392,27 @@ std::size_t TestFixture::runTests(const options& args)
     countTests = 0;
     errmsg.str("");
 
+    const auto& which_tests = args.which_tests();
+    const auto exclude_tests = args.exclude_tests();
+
     // TODO: bail out when given class/test is not found?
-    for (std::string classname : args.which_test()) {
-        std::string testname;
-        const std::string::size_type pos = classname.find("::");
-        if (pos != std::string::npos) {
-            // TODO: excluding indiviual tests is not supported yet
-            testname = classname.substr(pos + 2);
-            classname.erase(pos);
+    for (TestInstance * test : TestRegistry::theInstance().tests())
+    {
+        std::set<std::string> tests;
+        if (!which_tests.empty()) {
+            const auto it = which_tests.find(test->classname);
+            const bool match = it != which_tests.cend();
+            if (match && exclude_tests && it->second.empty()) // only bailout when the whole fixture is excluded
+                continue;
+            if (!match && !exclude_tests)
+                continue;
+            if (match)
+                tests = it->second;
         }
 
-        for (TestInstance * test : TestRegistry::theInstance().tests()) {
-            if (!classname.empty()) {
-                const bool match = test->classname == classname;
-                if ((match && args.exclude_tests()) || (!match && !args.exclude_tests()))
-                    continue;
-            }
-
-            TestFixture* fixture = test->create();
-            fixture->processOptions(args);
-            fixture->run(testname);
-        }
+        TestFixture* fixture = test->create();
+        fixture->processOptions(args);
+        fixture->run(tests);
     }
 
     if (args.summary() && !args.dry_run()) {

test/fixture.h

@@ -34,6 +34,7 @@
 #include <exception>
 #include <list>
 #include <memory>
+#include <set>
 #include <sstream>
 #include <stdexcept>
 #include <string>
@@ -56,9 +57,10 @@ class TestFixture : public ErrorLogger {
 
 protected:
     std::string exename;
-    std::string testToRun;
+    std::set<std::string> testsToRun;
     bool quiet_tests{};
     bool dry_run{};
+    bool exclude_tests{};
     bool mNewTemplate{};
 
     virtual void run() = 0;
@@ -290,7 +292,7 @@ class TestFixture : public ErrorLogger {
     {
         (void) metric;
     }
-    void run(const std::string &str);
+    void run(const std::set<std::string> &tests);
 
 public:
     static void printHelp();

test/options.cpp

@@ -17,23 +17,30 @@
 #include "options.h"
 
 options::options(int argc, const char* const argv[])
-    : mWhichTests(argv + 1, argv + argc)
-    ,mQuiet(mWhichTests.count("-q") != 0)
-    ,mHelp(mWhichTests.count("-h") != 0 || mWhichTests.count("--help"))
-    ,mSummary(mWhichTests.count("-n") == 0)
-    ,mDryRun(mWhichTests.count("-d") != 0)
-    ,mExcludeTests(mWhichTests.count("-x") != 0)
+    : mArgs(argv + 1, argv + argc)
+    ,mQuiet(mArgs.count("-q") != 0)
+    ,mHelp(mArgs.count("-h") != 0 || mArgs.count("--help"))
+    ,mSummary(mArgs.count("-n") == 0)
+    ,mDryRun(mArgs.count("-d") != 0)
+    ,mExcludeTests(mArgs.count("-x") != 0)
     ,mExe(argv[0])
 {
-    for (auto it = mWhichTests.cbegin(); it != mWhichTests.cend();) {
-        if (!it->empty() && (((*it)[0] == '-') || (it->find("::") != std::string::npos && mWhichTests.count(it->substr(0, it->find("::"))))))
-            it = mWhichTests.erase(it);
-        else
-            ++it;
-    }
-
-    if (mWhichTests.empty()) {
-        mWhichTests.insert("");
+    for (const auto& arg : mArgs) {
+        if (arg.empty())
+            continue; // empty argument
+        if (arg[0] == '-')
+            continue; // command-line switch
+        const auto pos = arg.find("::");
+        if (pos == std::string::npos) {
+            mWhichTests[arg] = {}; // run whole fixture
+            continue;
+        }
+        const std::string fixture = arg.substr(0, pos);
+        const auto it = mWhichTests.find(fixture);
+        if (it != mWhichTests.cend() && it->second.empty())
+            continue; // whole fixture is already included
+        const std::string test = arg.substr(pos+2);
+        mWhichTests[fixture].emplace(test); // run individual test
     }
 }
 
@@ -57,7 +64,7 @@ bool options::dry_run() const
     return mDryRun;
 }
 
-const std::set<std::string>& options::which_test() const
+const std::map<std::string, std::set<std::string>>& options::which_tests() const
 {
     return mWhichTests;
 }

test/options.h

@@ -17,6 +17,7 @@
 #ifndef OPTIONS_H
 #define OPTIONS_H
 
+#include <map>
 #include <set>
 #include <string>
 
@@ -39,8 +40,8 @@ class options {
     bool dry_run() const;
     /** Exclude provided lists of tests. */
     bool exclude_tests() const;
-    /** Which test should be run. Empty string means 'all tests' */
-    const std::set<std::string>& which_test() const;
+    /** Which tests should be run. */
+    const std::map<std::string, std::set<std::string>>& which_tests() const;
 
     const std::string& exe() const;
 
@@ -49,7 +50,8 @@ class options {
     options& operator =(const options&) = delete;
 
 private:
-    std::set<std::string> mWhichTests;
+    std::set<std::string> mArgs;
+    std::map<std::string, std::set<std::string>> mWhichTests;
     const bool mQuiet;
     const bool mHelp;
     const bool mSummary;

test/testoptions.cpp

@@ -51,21 +51,28 @@ class TestOptions : public TestFixture {
     void which_test() const {
         const char* argv[] = {"./test_runner", "TestClass"};
         options args(getArrayLength(argv), argv);
-        ASSERT(std::set<std::string> {"TestClass"} == args.which_test());
+        const std::map<std::string, std::set<std::string>> expected{
+            { "TestClass", {} }
+        };
+        ASSERT(expected == args.which_tests());
     }
 
 
     void which_test_method() const {
         const char* argv[] = {"./test_runner", "TestClass::TestMethod"};
         options args(getArrayLength(argv), argv);
-        ASSERT(std::set<std::string> {"TestClass::TestMethod"} == args.which_test());
+        const std::map<std::string, std::set<std::string>> expected{
+            { "TestClass", {"TestMethod"} }
+        };
+        ASSERT(expected == args.which_tests());
     }
 
 
     void no_test_method() const {
         const char* argv[] = {"./test_runner"};
         options args(getArrayLength(argv), argv);
-        ASSERT(std::set<std::string> {""} == args.which_test());
+        const std::map<std::string, std::set<std::string>> expected{};
+        ASSERT(expected == args.which_tests());
     }
 
 
@@ -105,22 +112,28 @@ class TestOptions : public TestFixture {
     void multiple_testcases() const {
         const char* argv[] = {"./test_runner", "TestClass::TestMethod", "TestClass::AnotherTestMethod"};
         options args(getArrayLength(argv), argv);
-        std::set<std::string> expected {"TestClass::TestMethod", "TestClass::AnotherTestMethod"};
-        ASSERT(expected == args.which_test());
+        const std::map<std::string, std::set<std::string>> expected{
+            { "TestClass", { "TestMethod", "AnotherTestMethod" } }
+        };
+        ASSERT(expected == args.which_tests());
     }
 
     void multiple_testcases_ignore_duplicates() const {
         const char* argv[] = {"./test_runner", "TestClass::TestMethod", "TestClass"};
         options args(getArrayLength(argv), argv);
-        std::set<std::string> expected {"TestClass"};
-        ASSERT(expected == args.which_test());
+        const std::map<std::string, std::set<std::string>> expected{
+            { "TestClass", {} }
+        };
+        ASSERT(expected == args.which_tests());
     }
 
     void invalid_switches() const {
         const char* argv[] = {"./test_runner", "TestClass::TestMethod", "-a", "-v", "-q"};
         options args(getArrayLength(argv), argv);
-        std::set<std::string> expected {"TestClass::TestMethod"};
-        ASSERT(expected == args.which_test());
+        const std::map<std::string, std::set<std::string>> expected {
+            { "TestClass", { "TestMethod" } }
+        };
+        ASSERT(expected == args.which_tests());
         ASSERT_EQUALS(true, args.quiet());
     }