mtest: TestResult.SKIP is not a failure (mesonbuild#7525)

* mtest: TestResult.SKIP is not a failure

If some but not all tests in a run were skipped, then the overall result
is given by whether there were any failures among the non-skipped tests.

Resolves: mesonbuild#7515
Signed-off-by: Simon McVittie <smcv@debian.org>

* Add test-cases for partially skipped TAP tests

issue7515.txt is the output of one of the real TAP tests in gjs, which
failed as a result of mesonbuild#7515. The version inline in meson.build is
a minimal reproducer.

Signed-off-by: Simon McVittie <smcv@debian.org>

mesonbuild/mtest.py

@@ -489,7 +489,7 @@ def make_tap(cls, test: 'TestSerialisation', test_env: T.Dict[str, str],
                 failed = True
             elif isinstance(i, TAPParser.Test):
                 results.append(i.result)
-                if i.result not in {TestResult.OK, TestResult.EXPECTEDFAIL}:
+                if i.result not in {TestResult.OK, TestResult.EXPECTEDFAIL, TestResult.SKIP}:
                     failed = True
             elif isinstance(i, TAPParser.Error):
                 results.append(TestResult.ERROR)

test cases/common/213 tap tests/cat.c

@@ -0,0 +1,26 @@
+#include <errno.h>
+#include <stdio.h>
+
+int main(int argc, char **argv) {
+    char buf[1024];
+    size_t len;
+    FILE *fh;
+
+    if (argc != 2) {
+        fprintf(stderr, "Incorrect number of arguments, got %i\n", argc);
+        return 1;
+    }
+    fh = fopen(argv[1], "r");
+    if (fh == NULL) {
+        fprintf(stderr, "Opening %s: errno=%i\n", argv[1], errno);
+        return 1;
+    }
+    do {
+        len = fread(buf, 1, sizeof(buf), fh);
+        if (len > 0) {
+            fwrite(buf, 1, len, stdout);
+        }
+    } while (len > 0);
+    fclose(fh);
+    return 0;
+}

test cases/common/213 tap tests/issue7515.txt

@@ -0,0 +1,27 @@
+1..26
+ok 1 Gtk overrides UI template sets up internal and public template children
+ok 2 Gtk overrides UI template sets up public template children with the correct widgets
+ok 3 Gtk overrides UI template sets up internal template children with the correct widgets
+ok 4 Gtk overrides UI template connects template callbacks to the correct handler
+ok 5 Gtk overrides UI template binds template callbacks to the correct object
+ok 6 Gtk overrides UI template from resource sets up internal and public template children
+ok 7 Gtk overrides UI template from resource sets up public template children with the correct widgets
+ok 8 Gtk overrides UI template from resource sets up internal template children with the correct widgets
+ok 9 Gtk overrides UI template from resource connects template callbacks to the correct handler
+ok 10 Gtk overrides UI template from resource binds template callbacks to the correct object
+ok 11 Gtk overrides UI template from file sets up internal and public template children
+ok 12 Gtk overrides UI template from file sets up public template children with the correct widgets
+ok 13 Gtk overrides UI template from file sets up internal template children with the correct widgets
+ok 14 Gtk overrides UI template from file connects template callbacks to the correct handler
+ok 15 Gtk overrides UI template from file binds template callbacks to the correct object
+ok 16 Gtk overrides Class inheriting from template class sets up internal and public template children # SKIP pending
+ok 17 Gtk overrides Class inheriting from template class sets up public template children with the correct widgets # SKIP pending
+ok 18 Gtk overrides Class inheriting from template class sets up internal template children with the correct widgets # SKIP pending
+ok 19 Gtk overrides Class inheriting from template class connects template callbacks to the correct handler # SKIP pending
+ok 20 Gtk overrides Class inheriting from template class binds template callbacks to the correct object # SKIP pending
+ok 21 Gtk overrides sets CSS names on classes
+ok 22 Gtk overrides avoid crashing when GTK vfuncs are called in garbage collection
+ok 23 Gtk overrides accepts string in place of GdkAtom
+ok 24 Gtk overrides accepts null in place of GdkAtom as GDK_NONE
+ok 25 Gtk overrides uses the correct GType for null child properties
+ok 26 Gtk overrides can create a Gtk.TreeIter with accessible stamp field

test cases/common/213 tap tests/meson.build

@@ -1,10 +1,13 @@
 project('test features', 'c')
 
 tester = executable('tester', 'tester.c')
+cat = executable('cat', 'cat.c')
 test('pass', tester, args : ['ok'], protocol: 'tap')
 test('fail', tester, args : ['not ok'], should_fail: true, protocol: 'tap')
 test('xfail', tester, args : ['not ok # todo'], protocol: 'tap')
 test('xpass', tester, args : ['ok # todo'], should_fail: true, protocol: 'tap')
 test('skip', tester, args : ['ok # skip'], protocol: 'tap')
+test('partially skipped', tester, args : ['ok 1\nok 2 # skip'], protocol: 'tap')
+test('partially skipped (real-world example)', cat, args : [files('issue7515.txt')], protocol: 'tap')
 test('skip failure', tester, args : ['not ok # skip'], should_fail: true, protocol: 'tap')
 test('no tests', tester, args : ['1..0 # skip'], protocol: 'tap')

test cases/failing test/5 tap tests/meson.build

@@ -4,3 +4,4 @@ tester = executable('tester', 'tester.c')
 test('nonzero return code', tester, args : [], protocol: 'tap')
 test('missing test', tester, args : ['1..1'], protocol: 'tap')
 test('incorrect skip', tester, args : ['1..1 # skip\nok 1'], protocol: 'tap')
+test('partially skipped', tester, args : ['not ok 1\nok 2 # skip'], protocol: 'tap')