[Unittests] Added a meta-test for tvm.testing.fixture behavior in case of a broken fixture. (apache#8343)

In these cases, the test function should be marked as failing the
setup, and should not run.  This is pytest's default behavior, and
should work whether or not a fixture is cached.

Co-authored-by: Eric Lunderberg <elunderberg@octoml.ai>

Author: Lunderberg

tests/python/unittest/test_tvm_testing_features.py

@@ -25,7 +25,10 @@
 # This file tests features in tvm.testing, such as verifying that
 # cached fixtures are run an appropriate number of times.  As a
 # result, the order of the tests is important.  Use of --last-failed
-# or --failed-first while debugging this file is not advised.
+# or --failed-first while debugging this file is not advised.  If
+# these tests are distributed/parallelized using pytest-xdist or
+# similar, all tests in this file should run sequentially on the same
+# node.  (See https://stackoverflow.com/a/59504228)
 
 
 class TestTargetAutoParametrization:
@@ -145,5 +148,37 @@ def test_cached_count(self):
             assert self.cached_calls == len(self.param1_vals)
 
 
+class TestBrokenFixture:
+    # Tests that use a fixture that throws an exception fail, and are
+    # marked as setup failures.  The tests themselves are never run.
+    # This behavior should be the same whether or not the fixture
+    # results are cached.
+
+    num_uses_broken_uncached_fixture = 0
+    num_uses_broken_cached_fixture = 0
+
+    @tvm.testing.fixture
+    def broken_uncached_fixture(self):
+        raise RuntimeError("Intentionally broken fixture")
+
+    @pytest.mark.xfail(True, reason="Broken fixtures should result in a failing setup", strict=True)
+    def test_uses_broken_uncached_fixture(self, broken_uncached_fixture):
+        type(self).num_uses_broken_fixture += 1
+
+    def test_num_uses_uncached(self):
+        assert self.num_uses_broken_uncached_fixture == 0
+
+    @tvm.testing.fixture(cache_return_value=True)
+    def broken_cached_fixture(self):
+        raise RuntimeError("Intentionally broken fixture")
+
+    @pytest.mark.xfail(True, reason="Broken fixtures should result in a failing setup", strict=True)
+    def test_uses_broken_cached_fixture(self, broken_cached_fixture):
+        type(self).num_uses_broken_cached_fixture += 1
+
+    def test_num_uses_cached(self):
+        assert self.num_uses_broken_cached_fixture == 0
+
+
 if __name__ == "__main__":
     sys.exit(pytest.main(sys.argv))