diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst new file mode 100644 index 0000000000..2e71a180bd --- /dev/null +++ b/hypothesis-python/RELEASE.rst @@ -0,0 +1,3 @@ +RELEASE_TYPE: minor + +Hypothesis now detects if it is running on a CI server and provides better default settings for running on CI in this case. diff --git a/hypothesis-python/docs/settings.rst b/hypothesis-python/docs/settings.rst index 5b2db744fb..7a2a0557d6 100644 --- a/hypothesis-python/docs/settings.rst +++ b/hypothesis-python/docs/settings.rst @@ -250,6 +250,14 @@ by your conftest you can load one with the command line option ``--hypothesis-pr $ pytest tests --hypothesis-profile +Hypothesis comes with two built-in profiles, ``ci`` and ``default``. +``ci`` is set up to have good defaults for running in a CI environment, so emphasizes determinism, while the +``default`` settings are picked to be more likely to find bugs and to have a good workflow when used for local development. + +Hypothesis will automatically detect certain common CI environments and use the CI profile automatically +when running in them. +In particular, if you wish to use the ``ci`` profile, setting the ``CI`` environment variable will do this. + .. _healthchecks: ------------- diff --git a/hypothesis-python/src/hypothesis/_settings.py b/hypothesis-python/src/hypothesis/_settings.py index 2f18ff920f..6aab3fa188 100644 --- a/hypothesis-python/src/hypothesis/_settings.py +++ b/hypothesis-python/src/hypothesis/_settings.py @@ -63,6 +63,7 @@ def __get__(self, obj, type=None): from hypothesis.database import ExampleDatabase result = ExampleDatabase(not_set) + assert result is not not_set return result except KeyError: raise AttributeError(self.name) from None @@ -407,6 +408,8 @@ def _max_examples_validator(x): :ref:`separate settings profiles ` - for example running quick deterministic tests on every commit, and a longer non-deterministic nightly testing run. + +By default when running on CI, this will be set to True. """, ) @@ -682,6 +685,8 @@ def _validate_deadline(x): variability in test run time). Set this to ``None`` to disable this behaviour entirely. + +By default when running on CI, this will be set to None. """, ) @@ -694,13 +699,11 @@ def is_in_ci() -> bool: settings._define_setting( "print_blob", - default=is_in_ci(), - show_default=False, + default=False, options=(True, False), description=""" If set to ``True``, Hypothesis will print code for failing examples that can be used with :func:`@reproduce_failure ` to reproduce the failing example. -The default is ``True`` if the ``CI`` or ``TF_BUILD`` env vars are set, ``False`` otherwise. """, ) @@ -750,6 +753,24 @@ def note_deprecation( settings.register_profile("default", settings()) settings.load_profile("default") + +assert settings.default is not None + +CI = settings( + derandomize=True, + deadline=None, + database=None, + print_blob=True, + suppress_health_check=[HealthCheck.too_slow], +) + +settings.register_profile("ci", CI) + + +# This is tested in a subprocess so the branch doesn't show up in coverage. +if is_in_ci(): # pragma: no cover + settings.load_profile("ci") + assert settings.default is not None diff --git a/hypothesis-python/src/hypothesis/core.py b/hypothesis-python/src/hypothesis/core.py index 9a1a19c1e2..188fb6d1a4 100644 --- a/hypothesis-python/src/hypothesis/core.py +++ b/hypothesis-python/src/hypothesis/core.py @@ -1564,23 +1564,23 @@ def wrapped_test(*arguments, **kwargs): "to ensure that each example is run in a separate " "database transaction." ) - if settings.database is not None: - nonlocal prev_self - # Check selfy really is self (not e.g. a mock) before we health-check - cur_self = ( - stuff.selfy - if getattr(type(stuff.selfy), test.__name__, None) is wrapped_test - else None + + nonlocal prev_self + # Check selfy really is self (not e.g. a mock) before we health-check + cur_self = ( + stuff.selfy + if getattr(type(stuff.selfy), test.__name__, None) is wrapped_test + else None + ) + if prev_self is Unset: + prev_self = cur_self + elif cur_self is not prev_self: + msg = ( + f"The method {test.__qualname__} was called from multiple " + "different executors. This may lead to flaky tests and " + "nonreproducible errors when replaying from database." ) - if prev_self is Unset: - prev_self = cur_self - elif cur_self is not prev_self: - msg = ( - f"The method {test.__qualname__} was called from multiple " - "different executors. This may lead to flaky tests and " - "nonreproducible errors when replaying from database." - ) - fail_health_check(settings, msg, HealthCheck.differing_executors) + fail_health_check(settings, msg, HealthCheck.differing_executors) state = StateForActualGivenExecution( stuff, test, settings, random, wrapped_test @@ -1675,7 +1675,6 @@ def wrapped_test(*arguments, **kwargs): # The exception caught here should either be an actual test # failure (or BaseExceptionGroup), or some kind of fatal error # that caused the engine to stop. - generated_seed = wrapped_test._hypothesis_internal_use_generated_seed with local_settings(settings): if not (state.failed_normally or generated_seed is None): diff --git a/hypothesis-python/tests/common/setup.py b/hypothesis-python/tests/common/setup.py index 8695c334cd..8881985137 100644 --- a/hypothesis-python/tests/common/setup.py +++ b/hypothesis-python/tests/common/setup.py @@ -12,7 +12,7 @@ from warnings import filterwarnings from hypothesis import HealthCheck, Phase, Verbosity, settings -from hypothesis._settings import not_set +from hypothesis._settings import CI, is_in_ci, not_set from hypothesis.internal.conjecture.data import AVAILABLE_PROVIDERS from hypothesis.internal.coverage import IN_COVERAGE_TESTS @@ -45,13 +45,14 @@ def run(): v = getattr(x, s.name) # Check if it has a dynamically defined default and if so skip comparison. if getattr(settings, s.name).show_default: - assert ( - v == s.default + assert v == s.default or ( + is_in_ci() and v == getattr(CI, s.name) ), f"({v!r} == x.{s.name}) != (s.{s.name} == {s.default!r})" settings.register_profile( "default", settings( + settings.get_profile("default"), max_examples=20 if IN_COVERAGE_TESTS else not_set, phases=list(Phase), # Dogfooding the explain phase ), diff --git a/hypothesis-python/tests/cover/test_settings.py b/hypothesis-python/tests/cover/test_settings.py index d85ced33cf..4e58926793 100644 --- a/hypothesis-python/tests/cover/test_settings.py +++ b/hypothesis-python/tests/cover/test_settings.py @@ -9,6 +9,7 @@ # obtain one at https://mozilla.org/MPL/2.0/. import datetime +import os import subprocess import sys from unittest import TestCase @@ -25,7 +26,7 @@ note_deprecation, settings, ) -from hypothesis.database import ExampleDatabase +from hypothesis.database import ExampleDatabase, InMemoryExampleDatabase from hypothesis.errors import ( HypothesisDeprecationWarning, InvalidArgument, @@ -108,12 +109,13 @@ def test_can_not_set_verbosity_to_non_verbosity(): @pytest.mark.parametrize("db", [None, ExampleDatabase()]) def test_inherits_an_empty_database(db): - assert settings.default.database is not None - s = settings(database=db) - assert s.database is db - with local_settings(s): - t = settings() - assert t.database is db + with local_settings(settings(database=InMemoryExampleDatabase())): + assert settings.default.database is not None + s = settings(database=db) + assert s.database is db + with local_settings(s): + t = settings() + assert t.database is db @pytest.mark.parametrize("db", [None, ExampleDatabase()]) @@ -273,6 +275,7 @@ def test_settings_as_decorator_must_be_on_callable(): from hypothesis.configuration import set_hypothesis_home_dir from hypothesis.database import DirectoryBasedExampleDatabase +settings.load_profile("default") settings.default.database if __name__ == '__main__': @@ -476,8 +479,12 @@ def __repr__(self): assert "parent=(not settings repr)" in str(excinfo.value) +def test_default_settings_do_not_use_ci(): + assert settings.get_profile("default").suppress_health_check == () + + def test_show_changed(): - s = settings(max_examples=999, database=None) + s = settings(settings.get_profile("default"), max_examples=999, database=None) assert s.show_changed() == "database=None, max_examples=999" @@ -511,3 +518,43 @@ def test_deprecated_settings_not_in_settings_all_list(): assert al == ls assert HealthCheck.return_value not in ls assert HealthCheck.not_a_test_method not in ls + + +@skipif_emscripten +def test_check_defaults_to_derandomize_when_running_on_ci(): + env = dict(os.environ) + env["CI"] = "true" + + assert ( + subprocess.check_output( + [ + sys.executable, + "-c", + "from hypothesis import settings\nprint(settings().derandomize)", + ], + env=env, + text=True, + encoding="utf-8", + ).strip() + == "True" + ) + + +@skipif_emscripten +def test_check_defaults_to_randomize_when_not_running_on_ci(): + env = dict(os.environ) + env.pop("CI", None) + env.pop("TF_BUILD", None) + assert ( + subprocess.check_output( + [ + sys.executable, + "-c", + "from hypothesis import settings\nprint(settings().derandomize)", + ], + env=env, + text=True, + encoding="utf-8", + ).strip() + == "False" + ) diff --git a/hypothesis-python/tests/pytest/test_capture.py b/hypothesis-python/tests/pytest/test_capture.py index f2765dd3a3..d5fb88ed9e 100644 --- a/hypothesis-python/tests/pytest/test_capture.py +++ b/hypothesis-python/tests/pytest/test_capture.py @@ -93,7 +93,8 @@ def test_healthcheck_traceback_is_hidden(x): """ -def test_healthcheck_traceback_is_hidden(testdir): +def test_healthcheck_traceback_is_hidden(testdir, monkeypatch): + monkeypatch.delenv("CI", raising=False) script = testdir.makepyfile(TRACEBACKHIDE_HEALTHCHECK) result = testdir.runpytest(script, "--verbose") def_token = "__ test_healthcheck_traceback_is_hidden __" diff --git a/hypothesis-python/tests/pytest/test_seeding.py b/hypothesis-python/tests/pytest/test_seeding.py index 04a61eec35..954628596e 100644 --- a/hypothesis-python/tests/pytest/test_seeding.py +++ b/hypothesis-python/tests/pytest/test_seeding.py @@ -82,7 +82,10 @@ def test_failure(i): """ -def test_repeats_healthcheck_when_following_seed_instruction(testdir, tmp_path): +def test_repeats_healthcheck_when_following_seed_instruction( + testdir, tmp_path, monkeypatch +): + monkeypatch.delenv("CI", raising=False) health_check_test = HEALTH_CHECK_FAILURE.replace( "", repr(str(tmp_path / "seen")) )