Add note to NonInteracteExampleWarning in selftests

Author: jobh

hypothesis-python/tests/conftest.py

@@ -17,6 +17,8 @@
 import pytest
 
 from hypothesis._settings import is_in_ci
+from hypothesis.errors import NonInteractiveExampleWarning
+from hypothesis.internal.compat import add_note
 from hypothesis.internal.detection import is_hypothesis_test
 
 from tests.common import TIME_INCREMENT
@@ -106,20 +108,20 @@ def pytest_runtest_call(item):
     # This hookwrapper checks for PRNG state leaks from Hypothesis tests.
     # See: https://github.com/HypothesisWorks/hypothesis/issues/1919
     if not (hasattr(item, "obj") and is_hypothesis_test(item.obj)):
-        yield
+        outcome = yield
     elif "pytest_randomly" in sys.modules:
         # See https://github.com/HypothesisWorks/hypothesis/issues/3041 - this
         # branch exists to make it easier on external contributors, but should
         # never run in our CI (because that would disable the check entirely).
         assert not is_in_ci()
-        yield
+        outcome = yield
     else:
         # We start by peturbing the state of the PRNG, because repeatedly
         # leaking PRNG state resets state_after to the (previously leaked)
         # state_before, and that just shows as "no use of random".
         random.seed(independent_random.randrange(2**32))
         before = random.getstate()
-        yield
+        outcome = yield
         after = random.getstate()
         if before != after:
             if after in random_states_after_tests:
@@ -129,3 +131,11 @@ def pytest_runtest_call(item):
                     "same global `random.getstate()`; this is probably a nasty bug!"
                 )
             random_states_after_tests[after] = item.nodeid
+
+    # Annotate usage of .example() with a hint about alternatives
+    if isinstance(outcome.exception, NonInteractiveExampleWarning):
+        add_note(
+            outcome.exception,
+            "For hypothesis' own test suite, consider using one of the helper "
+            "methods in tests.common.debug instead.",
+        )

hypothesis-python/tests/cover/test_interactive_example.py

@@ -27,6 +27,8 @@
 from tests.common.debug import find_any
 from tests.common.utils import fails_with
 
+pytest_plugins = "pytester"
+
 
 # Allow calling .example() without warnings for all tests in this module
 @pytest.fixture(scope="function", autouse=True)
@@ -84,6 +86,27 @@ def test_non_interactive_example_emits_warning():
             st.text().example()
 
 
+EXAMPLE_GENERATING_TEST = """
+from hypothesis import strategies as st
+
+def test_interactive_example():
+    st.integers().example()
+"""
+
+
+def test_selftests_exception_contains_note(pytester):
+    # The note is added by a pytest hook, so we need to run it under pytest in a
+    # subenvironment with (effectively) the same toplevel conftest.
+    with warnings.catch_warnings():
+        warnings.simplefilter("error")
+
+        pytester.makeconftest("from tests.conftest import *")
+        result = pytester.runpytest_inprocess(
+            pytester.makepyfile(EXAMPLE_GENERATING_TEST)
+        )
+        assert "helper methods in tests.common.debug" in "\n".join(result.outlines)
+
+
 @pytest.mark.skipif(WINDOWS, reason="pexpect.spawn not supported on Windows")
 def test_interactive_example_does_not_emit_warning():
     try: