Add --list-test-cases and --run-test-case

These options allow an external runner (such as [test-run][test-run]) to
run the test cases separately: isolated and/or parallel.

The idea is the following.

* test-run has some criteria, whether to parallelize the given test.
* If it needs to be parallelized, test-run invokes `luatest
  --list-test-cases <..path to test..>`.
* Then each test case is run in its own process using `luatest
  --run-test-case <..test case..> <..path to test..>`.

The `--run-test-case` option may be simulated using `--pattern`, but it
requires regexp escaping. It is more convenient to have a separate
option to pass a verbatim test case name.

The new CLI options are mostly for scripting and not for interactive
use. So, no short option are added for them.

[test-run] https://github.com/tarantool/test-run

Author: Totktonada

CHANGELOG.md

@@ -11,6 +11,7 @@
 - Changed error message for too long Unix domain socket paths (gh-341).
 - Add `cbuilder` helper as a declarative configuration builder (gh-366).
 - Make `assert_error_*` additionally check error trace if required.
+- Add `--list-test-cases` and `--run-test-case` CLI options.
 
 ## 1.0.1
 

luatest/runner.lua

@@ -161,6 +161,9 @@ Options:
   --coverage:             Use luacov to collect code coverage.
   --no-clean:             Disable the var directory (default: /tmp/t) deletion before
                           running tests.
+  --list-test-cases       List all test cases.
+  --run-test-case CASE    Run one specific test case. Unlike --pattern the name
+                          should match verbatim.
 ]]
 
 function Runner.parse_cmd_line(args)
@@ -236,6 +239,10 @@ function Runner.parse_cmd_line(args)
             result.coverage_report = true
         elseif arg == '--no-clean' then
             result.no_clean = true
+        elseif arg == '--list-test-cases' then
+            result.list_test_cases = true
+        elseif arg == '--run-test-case' then
+            result.run_test_case = next_arg()
         elseif arg:sub(1,1) == '-' then
             error('Unknown option: ' .. arg)
         elseif arg:find('/') then
@@ -284,10 +291,16 @@ function Runner.is_test_name(s)
     return string.sub(s, 1, 4):lower() == 'test'
 end
 
-function Runner.filter_tests(tests, patterns)
+function Runner.filter_tests(tests, patterns, test_case)
     local result = {[true] = {}, [false] = {}}
     for _, test in ipairs(tests) do
-        table.insert(result[utils.pattern_filter(patterns, test.name)], test)
+        -- Handle --pattern CLI option.
+        local yesno = utils.pattern_filter(patterns, test.name)
+        -- Handle --run-test-case CLI option.
+        if test_case ~= nil then
+            yesno = yesno and test.name == test_case
+        end
+        table.insert(result[yesno], test)
     end
     return result
 end
@@ -349,7 +362,17 @@ end
 
 function Runner.mt:run()
     self:bootstrap()
-    local filtered_list = self.class.filter_tests(self:find_tests(), self.tests_pattern)
+    local filtered_list = self.class.filter_tests(self:find_tests(),
+        self.tests_pattern, self.run_test_case)
+
+    -- Handle the --list-test-case CLI option.
+    if self.list_test_cases then
+        for _, test_case in ipairs(filtered_list[true]) do
+            print(test_case.name)
+        end
+        return 0
+    end
+
     self:start_suite(#filtered_list[true], #filtered_list[false])
     self:cleanup()
     self:run_tests(filtered_list[true])

test/luaunit/utility_test.lua

@@ -610,6 +610,12 @@ function g.test_parse_cmd_line()
         output_file_name='toto.xml',
     })
 
+    -- list-test-cases
+    assert_subject({'--list-test-cases'}, {list_test_cases = true})
+
+    -- run-test-case
+    assert_subject({'--run-test-case', 'foo'}, {run_test_case = 'foo'})
+
     t.assert_error_msg_contains('option: -$', subject, {'-$',})
 end
 
@@ -708,6 +714,21 @@ function g.test_filter_tests()
     included, excluded = subject(testset, {'foo', 'bar', '!t.t.', '%.bar'})
     t.assert_equals(included, {testset[2], testset[4], testset[6], testset[7], testset[8]})
     t.assert_equals(#excluded, 3)
+
+    -- --run-test-case without patterns
+    included, excluded = subject(testset, nil, 'toto.foo')
+    t.assert_equals(included, {testset[1]})
+    t.assert_equals(#excluded, 7)
+
+    -- --run-test-case with a matching pattern
+    included, excluded = subject(testset, {'toto'}, 'toto.foo')
+    t.assert_equals(included, {testset[1]})
+    t.assert_equals(#excluded, 7)
+
+    -- --run-test-case with a non-matching pattern
+    included, excluded = subject(testset, {'tutu'}, 'toto.foo')
+    t.assert_equals(included, {})
+    t.assert_equals(#excluded, 8)
 end
 
 function g.test_str_match()