From 4e25ca74cee011e15d797b026fcb56bd9799b672 Mon Sep 17 00:00:00 2001 From: Liam DeVoe Date: Mon, 10 Mar 2025 01:32:58 -0400 Subject: [PATCH 01/24] generate subtree mutations --- hypothesis-python/RELEASE.rst | 22 ++++ .../hypothesis/internal/conjecture/engine.py | 102 ++++++++++++++---- .../tests/conjecture/test_mutations.py | 48 +++++++++ .../tests/nocover/test_sampled_from.py | 9 +- .../tests/quality/test_poisoned_trees.py | 4 +- 5 files changed, 160 insertions(+), 25 deletions(-) create mode 100644 hypothesis-python/RELEASE.rst create mode 100644 hypothesis-python/tests/conjecture/test_mutations.py diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst new file mode 100644 index 0000000000..7a735d9e3d --- /dev/null +++ b/hypothesis-python/RELEASE.rst @@ -0,0 +1,22 @@ +RELEASE_TYPE: patch + +For strategies which draw make recursive draws, including :func:`~hypothesis.strategies.recursive` and :func:`~hypothesis.strategies.deferred`, we now generate examples with duplicated subtrees more often. This tends to uncover interesting behavior in tests. + +For instance, we might now generate a tree like this more often (though the details depend on the strategy): + +.. code-block:: none + + ┌─────┐ + ┌──────┤ a ├──────┐ + │ └─────┘ │ + ┌──┴──┐ ┌──┴──┐ + │ b │ │ a │ + └──┬──┘ └──┬──┘ + ┌────┴────┐ ┌────┴────┐ + ┌──┴──┐ ┌──┴──┐ ┌──┴──┐ ┌──┴──┐ + │ c │ │ d │ │ b │ │ ... │ + └─────┘ └─────┘ └──┬──┘ └─────┘ + ┌────┴────┐ + ┌──┴──┐ ┌──┴──┐ + │ c │ │ d │ + └─────┘ └─────┘ diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/engine.py b/hypothesis-python/src/hypothesis/internal/conjecture/engine.py index 0cbee69b7d..b76f911f37 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/engine.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/engine.py @@ -1162,27 +1162,82 @@ def generate_mutations_from( break group = self.random.choice(groups) - (start1, end1), (start2, end2) = self.random.sample(sorted(group), 2) - if (start1 <= start2 <= end2 <= end1) or ( - start2 <= start1 <= end1 <= end2 - ): # pragma: no cover # flaky on conjecture-cover tests - # one example entirely contains the other. give up. - # TODO use more intelligent mutation for containment, like - # replacing child with parent or vice versa. Would allow for - # recursive / subtree mutation - failed_mutations += 1 - continue - if start1 > start2: (start1, end1), (start2, end2) = (start2, end2), (start1, end1) - assert end1 <= start2 - - choices = data.choices - (start, end) = self.random.choice([(start1, end1), (start2, end2)]) - replacement = choices[start:end] - try: + if ( + start1 <= start2 <= end2 <= end1 + ): # pragma: no cover # flaky on conjecture-cover tests + # One span entirely contains the other. The strategy is very + # likely some kind of tree. e.g. we might have + # + # ┌─────┐ + # ┌─────┤ a ├──────┐ + # │ └─────┘ │ + # ┌──┴──┐ ┌──┴──┐ + # ┌──┤ b ├──┐ ┌──┤ c ├──┐ + # │ └──┬──┘ │ │ └──┬──┘ │ + # ┌─┴─┐ ┌─┴─┐ ┌─┴─┐ ┌─┴─┐ ┌─┴─┐ ┌─┴─┐ + # │ d │ │ e │ │ f │ │ g │ │ h │ │ i │ + # └───┘ └───┘ └───┘ └───┘ └───┘ └───┘ + # + # where each node is drawn from the same strategy and so + # has the same span label. We might have selected the spans + # corresponding to the a and c nodes, which is the entire + # tree and the subtree of (and including) c respectively. + # + # There are two possible mutations we could apply in this case: + # 1. replace a with c (replace child with parent) + # 2. replace c with a (replace parent with child) + # + # (1) results in multiple partial copies of the + # parent: + # ┌─────┐ + # ┌─────┤ a ├────────────┐ + # │ └─────┘ │ + # ┌──┴──┐ ┌─┴───┐ + # ┌──┤ b ├──┐ ┌─────┤ a ├──────┐ + # │ └──┬──┘ │ │ └─────┘ │ + # ┌─┴─┐ ┌─┴─┐ ┌─┴─┐ ┌──┴──┐ ┌──┴──┐ + # │ d │ │ e │ │ f │ ┌──┤ b ├──┐ ┌──┤ c ├──┐ + # └───┘ └───┘ └───┘ │ └──┬──┘ │ │ └──┬──┘ │ + # ┌─┴─┐ ┌─┴─┐ ┌─┴─┐ ┌─┴─┐ ┌─┴─┐ ┌─┴─┐ + # │ d │ │ e │ │ f │ │ g │ │ h │ │ i │ + # └───┘ └───┘ └───┘ └───┘ └───┘ └───┘ + # + # While (2) results in truncating part of the parent: + # + # ┌─────┐ + # ┌──┤ c ├──┐ + # │ └──┬──┘ │ + # ┌─┴─┐ ┌─┴─┐ ┌─┴─┐ + # │ g │ │ h │ │ i │ + # └───┘ └───┘ └───┘ + # + # (1) is the same as Example IV.4. in Nautilus (NDSS '19) + # (https://wcventure.github.io/FuzzingPaper/Paper/NDSS19_Nautilus.pdf), + # except we do not repeat the replacement additional times + # (the paper repeats it once for a total of two copies). + # + # We currently only apply mutation (1), and ignore mutation + # (2). The reason is that the attempt generated from (2) is + # always something that Hypothesis could easily have generated + # itself, by simply not making various choices. Whereas + # duplicating the exact value + structure of particular choices + # in (1) would have been hard for Hypothesis to generate by + # chance. + # + # TODO: an extension of this mutation might repeat (1) on + # a geometric distribution between 0 and ~10 times. We would + # need to find the corresponding span to recurse on in the new + # choices, probably just by using the choices index. + + # case (1): duplicate the choices in start1:start2. + attempt = data.choices[:start2] + data.choices[start1:] + else: + (start, end) = self.random.choice([(start1, end1), (start2, end2)]) + replacement = data.choices[start:end] # We attempt to replace both the examples with # whichever choice we made. Note that this might end # up messing up and getting the example boundaries @@ -1191,12 +1246,17 @@ def generate_mutations_from( # really matter. It may not achieve the desired result, # but it's still a perfectly acceptable choice sequence # to try. - new_data = self.cached_test_function( - choices[:start1] + attempt = ( + data.choices[:start1] + replacement - + choices[end1:start2] + + data.choices[end1:start2] + replacement - + choices[end2:], + + data.choices[end2:] + ) + + try: + new_data = self.cached_test_function( + attempt, # We set error_on_discard so that we don't end up # entering parts of the tree we consider redundant # and not worth exploring. diff --git a/hypothesis-python/tests/conjecture/test_mutations.py b/hypothesis-python/tests/conjecture/test_mutations.py new file mode 100644 index 0000000000..584773bb77 --- /dev/null +++ b/hypothesis-python/tests/conjecture/test_mutations.py @@ -0,0 +1,48 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +from hypothesis import strategies as st + +from tests.common.debug import find_any + +tree = st.deferred(lambda: st.tuples(st.integers(), tree, tree)) | st.just(None) + + +def test_can_find_duplicated_subtree(): + # look for an example of the form + # + # ┌─────┐ + # ┌──────┤ a ├──────┐ + # │ └─────┘ │ + # ┌──┴──┐ ┌──┴──┐ + # │ b │ │ a │ + # └──┬──┘ └──┬──┘ + # ┌────┴────┐ ┌────┴────┐ + # ┌──┴──┐ ┌──┴──┐ ┌──┴──┐ ┌──┴──┐ + # │ c │ │ d │ │ b │ │ ... │ + # └─────┘ └─────┘ └──┬──┘ └─────┘ + # ┌────┴────┐ + # ┌──┴──┐ ┌──┴──┐ + # │ c │ │ d │ + # └─────┘ └─────┘ + # + # If we just checked that (b, c, d) was duplicated somewhere, this could have + # happened as a result of normal mutation. Checking for the a parent node as + # well is unlikely to have been generated without tree mutation, however. + find_any( + tree, + ( + lambda v: v is not None + and v[1] is not None + and v[2] is not None + and v[0] == v[2][0] + and v[1] == v[2][1] + ), + ) diff --git a/hypothesis-python/tests/nocover/test_sampled_from.py b/hypothesis-python/tests/nocover/test_sampled_from.py index f72beef0ab..87db3b0c73 100644 --- a/hypothesis-python/tests/nocover/test_sampled_from.py +++ b/hypothesis-python/tests/nocover/test_sampled_from.py @@ -15,7 +15,7 @@ import pytest -from hypothesis import given, strategies as st +from hypothesis import given, settings, strategies as st from hypothesis.errors import InvalidArgument from hypothesis.internal.compat import bit_count from hypothesis.strategies._internal.strategies import SampledFromStrategy @@ -131,7 +131,12 @@ def test_flags_minimizes_bit_count(): def test_flags_finds_all_bits_set(): - assert find_any(st.sampled_from(LargeFlag), lambda f: f == ~LargeFlag(0)) + assert find_any( + st.sampled_from(LargeFlag), + lambda f: f == ~LargeFlag(0), + # see https://github.com/HypothesisWorks/hypothesis/pull/4295 + settings=settings(max_examples=10000), + ) def test_sample_unnamed_alias(): diff --git a/hypothesis-python/tests/quality/test_poisoned_trees.py b/hypothesis-python/tests/quality/test_poisoned_trees.py index f7cb005c24..969df21a20 100644 --- a/hypothesis-python/tests/quality/test_poisoned_trees.py +++ b/hypothesis-python/tests/quality/test_poisoned_trees.py @@ -36,8 +36,8 @@ def do_draw(self, data): return data.draw(self) + data.draw(self) else: # We draw n as two separate calls so that it doesn't show up as a - # single block. If it did, the heuristics that allow us to move - # blocks around would fire and it would move right, which would + # single choice. If it did, the heuristics that allow us to move + # choices around would fire and it would move right, which would # then allow us to shrink it more easily. n1 = data.draw_integer(0, 2**16 - 1) << 16 n2 = data.draw_integer(0, 2**16 - 1) From 7d45a1aa4c90628f44323b52f587da5916d067d6 Mon Sep 17 00:00:00 2001 From: Liam DeVoe Date: Mon, 10 Mar 2025 17:19:56 -0400 Subject: [PATCH 02/24] add calc_label to SampledFromStrategy --- .../src/hypothesis/internal/conjecture/utils.py | 4 ++++ .../src/hypothesis/strategies/_internal/misc.py | 4 ++-- .../hypothesis/strategies/_internal/strategies.py | 15 +++++++++++++-- hypothesis-python/tests/conjecture/test_utils.py | 5 +++++ .../tests/nocover/test_sampled_from.py | 9 ++------- 5 files changed, 26 insertions(+), 11 deletions(-) diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/utils.py b/hypothesis-python/src/hypothesis/internal/conjecture/utils.py index cf78bfeb99..a6c71cfbff 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/utils.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/utils.py @@ -38,6 +38,10 @@ def calc_label_from_cls(cls: type) -> int: return calc_label_from_name(cls.__qualname__) +def calc_label_from_hash(obj: object) -> int: + return calc_label_from_name(str(hash(obj))) + + def combine_labels(*labels: int) -> int: label = 0 for l in labels: diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/misc.py b/hypothesis-python/src/hypothesis/strategies/_internal/misc.py index 3d0b0c97e0..343c1d427f 100644 --- a/hypothesis-python/src/hypothesis/strategies/_internal/misc.py +++ b/hypothesis-python/src/hypothesis/strategies/_internal/misc.py @@ -13,7 +13,7 @@ SampledFromStrategy, SearchStrategy, T, - is_simple_data, + is_hashable, ) from hypothesis.strategies._internal.utils import cacheable, defines_strategy @@ -45,7 +45,7 @@ def __repr__(self): return f"just({get_pretty_function_description(self.value)}){suffix}" def calc_is_cacheable(self, recur): - return is_simple_data(self.value) + return is_hashable(self.value) def do_filtered_draw(self, data): # The parent class's `do_draw` implementation delegates directly to diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/strategies.py b/hypothesis-python/src/hypothesis/strategies/_internal/strategies.py index f32ac4b1b0..fd37c7a08a 100644 --- a/hypothesis-python/src/hypothesis/strategies/_internal/strategies.py +++ b/hypothesis-python/src/hypothesis/strategies/_internal/strategies.py @@ -41,6 +41,7 @@ from hypothesis.internal.conjecture.data import ConjectureData from hypothesis.internal.conjecture.utils import ( calc_label_from_cls, + calc_label_from_hash, calc_label_from_name, combine_labels, ) @@ -501,7 +502,7 @@ def do_draw(self, data: ConjectureData) -> Ex: raise NotImplementedError(f"{type(self).__name__}.do_draw") -def is_simple_data(value: object) -> bool: +def is_hashable(value: object) -> bool: try: hash(value) return True @@ -559,6 +560,16 @@ def __repr__(self) -> str: for name, f in self._transformations ) + def calc_label(self): + return combine_labels( + self.class_label, + *( + (calc_label_from_hash(self.elements),) + if is_hashable(self.elements) + else () + ), + ) + def calc_has_reusable_values(self, recur: RecurT) -> Any: # Because our custom .map/.filter implementations skip the normal # wrapper strategies (which would automatically return False for us), @@ -567,7 +578,7 @@ def calc_has_reusable_values(self, recur: RecurT) -> Any: return not self._transformations def calc_is_cacheable(self, recur: RecurT) -> Any: - return is_simple_data(self.elements) + return is_hashable(self.elements) def _transform( self, diff --git a/hypothesis-python/tests/conjecture/test_utils.py b/hypothesis-python/tests/conjecture/test_utils.py index 9b270ed119..9544e36e5b 100644 --- a/hypothesis-python/tests/conjecture/test_utils.py +++ b/hypothesis-python/tests/conjecture/test_utils.py @@ -106,6 +106,11 @@ def test_combine_labels_is_distinct(): assert cu.combine_labels(x, y) not in (x, y) +@given(st.integers()) +def test_combine_labels_is_identity_for_single_argument(n): + assert cu.combine_labels(n) == n + + @pytest.mark.skipif(np is None, reason="requires Numpy") def test_invalid_numpy_sample(): with pytest.raises(InvalidArgument): diff --git a/hypothesis-python/tests/nocover/test_sampled_from.py b/hypothesis-python/tests/nocover/test_sampled_from.py index 87db3b0c73..f72beef0ab 100644 --- a/hypothesis-python/tests/nocover/test_sampled_from.py +++ b/hypothesis-python/tests/nocover/test_sampled_from.py @@ -15,7 +15,7 @@ import pytest -from hypothesis import given, settings, strategies as st +from hypothesis import given, strategies as st from hypothesis.errors import InvalidArgument from hypothesis.internal.compat import bit_count from hypothesis.strategies._internal.strategies import SampledFromStrategy @@ -131,12 +131,7 @@ def test_flags_minimizes_bit_count(): def test_flags_finds_all_bits_set(): - assert find_any( - st.sampled_from(LargeFlag), - lambda f: f == ~LargeFlag(0), - # see https://github.com/HypothesisWorks/hypothesis/pull/4295 - settings=settings(max_examples=10000), - ) + assert find_any(st.sampled_from(LargeFlag), lambda f: f == ~LargeFlag(0)) def test_sample_unnamed_alias(): From 47ba4a614db0232c18f940d12804764f86b6b48b Mon Sep 17 00:00:00 2001 From: Liam DeVoe Date: Mon, 10 Mar 2025 18:08:15 -0400 Subject: [PATCH 03/24] add a urandom provider --- hypothesis-python/RELEASE.rst | 3 + .../internal/conjecture/providers.py | 65 +++++++++++++------ .../conjecture/test_provider_contract.py | 13 +++- 3 files changed, 59 insertions(+), 22 deletions(-) create mode 100644 hypothesis-python/RELEASE.rst diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst new file mode 100644 index 0000000000..bd9a6e58c0 --- /dev/null +++ b/hypothesis-python/RELEASE.rst @@ -0,0 +1,3 @@ +RELEASE_TYPE: patch + +This patch adds an :ref:`alternative backend ` which draws randomness from ``/dev/urandom``. This is useful for users of `Antithesis `_ who also have Hypothesis tests, allowing Antithesis mutation of ``/dev/urandom`` to drive Hypothesis generation. diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/providers.py b/hypothesis-python/src/hypothesis/internal/conjecture/providers.py index bf1609709c..8665a54b1f 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/providers.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/providers.py @@ -12,6 +12,7 @@ import contextlib import math from collections.abc import Iterable +from random import Random from sys import float_info from typing import ( TYPE_CHECKING, @@ -348,20 +349,20 @@ class HypothesisProvider(PrimitiveProvider): def __init__(self, conjecturedata: Optional["ConjectureData"], /): super().__init__(conjecturedata) + self._random = None if self._cd is None else self._cd._random def draw_boolean( self, p: float = 0.5, ) -> bool: - assert self._cd is not None - assert self._cd._random is not None + assert self._random is not None if p <= 0: return False if p >= 1: return True - return self._cd._random.random() < p + return self._random.random() < p def draw_integer( self, @@ -471,7 +472,7 @@ def draw_string( max_size: int = COLLECTION_DEFAULT_MAX_SIZE, ) -> str: assert self._cd is not None - assert self._cd._random is not None + assert self._random is not None if len(intervals) == 0: return "" @@ -501,11 +502,11 @@ def draw_string( while elements.more(): if len(intervals) > 256: if self.draw_boolean(0.2): - i = self._cd._random.randint(256, len(intervals) - 1) + i = self._random.randint(256, len(intervals) - 1) else: - i = self._cd._random.randint(0, 255) + i = self._random.randint(0, 255) else: - i = self._cd._random.randint(0, len(intervals) - 1) + i = self._random.randint(0, len(intervals) - 1) chars.append(intervals.char_in_shrink_order(i)) @@ -517,7 +518,7 @@ def draw_bytes( max_size: int = COLLECTION_DEFAULT_MAX_SIZE, ) -> bytes: assert self._cd is not None - assert self._cd._random is not None + assert self._random is not None buf = bytearray() average_size = min( @@ -532,25 +533,24 @@ def draw_bytes( observe=False, ) while elements.more(): - buf += self._cd._random.randbytes(1) + buf += self._random.randbytes(1) return bytes(buf) def _draw_float(self) -> float: - assert self._cd is not None - assert self._cd._random is not None + assert self._random is not None - f = lex_to_float(self._cd._random.getrandbits(64)) - sign = 1 if self._cd._random.getrandbits(1) else -1 + f = lex_to_float(self._random.getrandbits(64)) + sign = 1 if self._random.getrandbits(1) else -1 return sign * f def _draw_unbounded_integer(self) -> int: assert self._cd is not None - assert self._cd._random is not None + assert self._random is not None size = INT_SIZES[INT_SIZES_SAMPLER.sample(self._cd)] - r = self._cd._random.getrandbits(size) + r = self._random.getrandbits(size) sign = r & 1 r >>= 1 if sign: @@ -566,22 +566,22 @@ def _draw_bounded_integer( ) -> int: assert lower <= upper assert self._cd is not None - assert self._cd._random is not None + assert self._random is not None if lower == upper: return lower bits = (upper - lower).bit_length() - if bits > 24 and vary_size and self._cd._random.random() < 7 / 8: + if bits > 24 and vary_size and self._random.random() < 7 / 8: # For large ranges, we combine the uniform random distribution # with a weighting scheme with moderate chance. Cutoff at 2 ** 24 so that our # choice of unicode characters is uniform but the 32bit distribution is not. idx = INT_SIZES_SAMPLER.sample(self._cd) cap_bits = min(bits, INT_SIZES[idx]) upper = min(upper, lower + 2**cap_bits - 1) - return self._cd._random.randint(lower, upper) + return self._random.randint(lower, upper) - return self._cd._random.randint(lower, upper) + return self._random.randint(lower, upper) @classmethod def _draw_float_init_logic( @@ -827,3 +827,30 @@ def draw_bytes( ) -> bytes: values = self._draw_collection(min_size, max_size, alphabet_size=2**8) return bytes(values) + + +class URandom(Random): + # we reimplement a Random instance instead of using SystemRandom, because + # os.urandom is not guaranteed to read from /dev/urandom. + def getrandbits(self, k: int) -> int: + assert k >= 0 + size = bits_to_bytes(k) + with open("/dev/urandom", "rb") as f: + n = int_from_bytes(bytearray(f.read(size))) + # trim excess bits + return n >> (size * 8 - k) + + +class URandomProvider(HypothesisProvider): + # A provider which reads directly from /dev/urandom as its source of randomness. + # This provider exists to provide better Hypothesis integration with Antithesis + # (https://antithesis.com/), which interprets calls to /dev/urandom as the + # randomness to mutate. This effectively gives Antithesis control over + # the choices made by the URandomProvider. + # + # If you are not using Antithesis, you probably don't want to use this + # provider. + + def __init__(self, conjecturedata: Optional["ConjectureData"], /): + super().__init__(conjecturedata) + self._random = URandom() diff --git a/hypothesis-python/tests/conjecture/test_provider_contract.py b/hypothesis-python/tests/conjecture/test_provider_contract.py index 5b38544d69..1f107cf446 100644 --- a/hypothesis-python/tests/conjecture/test_provider_contract.py +++ b/hypothesis-python/tests/conjecture/test_provider_contract.py @@ -8,6 +8,8 @@ # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. +import pytest + from hypothesis import example, given, strategies as st from hypothesis.errors import StopTest from hypothesis.internal.conjecture.choice import ( @@ -16,7 +18,11 @@ choice_permitted, ) from hypothesis.internal.conjecture.data import ConjectureData -from hypothesis.internal.conjecture.providers import BytestringProvider +from hypothesis.internal.conjecture.providers import ( + BytestringProvider, + HypothesisProvider, + URandomProvider, +) from hypothesis.internal.intervalsets import IntervalSet from tests.conjecture.common import ( @@ -63,9 +69,10 @@ def test_provider_contract_bytestring(bytestring, choice_type_and_kwargs): ) +@pytest.mark.parametrize("provider", [URandomProvider, HypothesisProvider]) @given(st.lists(nodes()), st.randoms()) -def test_provider_contract_hypothesis(nodes, random): - data = ConjectureData(random=random) +def test_provider_contract(provider, nodes, random): + data = ConjectureData(random=random, provider=provider) for node in nodes: value = getattr(data, f"draw_{node.type}")(**node.kwargs) assert choice_permitted(value, node.kwargs) From 65c1440b0373304e1afa42f6cce47d1c60207b6e Mon Sep 17 00:00:00 2001 From: Liam DeVoe Date: Mon, 10 Mar 2025 18:41:11 -0400 Subject: [PATCH 04/24] skip on windows --- .../tests/conjecture/test_provider_contract.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/hypothesis-python/tests/conjecture/test_provider_contract.py b/hypothesis-python/tests/conjecture/test_provider_contract.py index 1f107cf446..2aba2019e9 100644 --- a/hypothesis-python/tests/conjecture/test_provider_contract.py +++ b/hypothesis-python/tests/conjecture/test_provider_contract.py @@ -12,6 +12,7 @@ from hypothesis import example, given, strategies as st from hypothesis.errors import StopTest +from hypothesis.internal.compat import WINDOWS from hypothesis.internal.conjecture.choice import ( choice_equal, choice_from_index, @@ -69,7 +70,18 @@ def test_provider_contract_bytestring(bytestring, choice_type_and_kwargs): ) -@pytest.mark.parametrize("provider", [URandomProvider, HypothesisProvider]) +@pytest.mark.parametrize( + "provider", + [ + pytest.param( + URandomProvider, + marks=pytest.mark.skipif( + WINDOWS, reason="/dev/urandom not available on windows" + ), + ), + HypothesisProvider, + ], +) @given(st.lists(nodes()), st.randoms()) def test_provider_contract(provider, nodes, random): data = ConjectureData(random=random, provider=provider) From b130af43e4c1dc58934c1805e47431e376ebba87 Mon Sep 17 00:00:00 2001 From: Liam DeVoe Date: Mon, 10 Mar 2025 21:26:40 -0400 Subject: [PATCH 05/24] register hypothesis-urandom, add to docs --- hypothesis-python/RELEASE.rst | 2 +- hypothesis-python/docs/strategies.rst | 39 ++++++++++++++----- .../internal/conjecture/providers.py | 16 +++++--- .../tests/conjecture/test_alt_backend.py | 19 ++++++++- 4 files changed, 58 insertions(+), 18 deletions(-) diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst index bd9a6e58c0..618a00cb32 100644 --- a/hypothesis-python/RELEASE.rst +++ b/hypothesis-python/RELEASE.rst @@ -1,3 +1,3 @@ -RELEASE_TYPE: patch +RELEASE_TYPE: minor This patch adds an :ref:`alternative backend ` which draws randomness from ``/dev/urandom``. This is useful for users of `Antithesis `_ who also have Hypothesis tests, allowing Antithesis mutation of ``/dev/urandom`` to drive Hypothesis generation. diff --git a/hypothesis-python/docs/strategies.rst b/hypothesis-python/docs/strategies.rst index 91fdb2521a..d3b0b81b52 100644 --- a/hypothesis-python/docs/strategies.rst +++ b/hypothesis-python/docs/strategies.rst @@ -241,25 +241,44 @@ Alternative backends for Hypothesis .. warning:: - EXPERIMENTAL AND UNSTABLE. + Alternative backends are experimental and not yet part of the public API. + We may continue to make breaking changes as we finalize the interface. -The importable name of a backend which Hypothesis should use to generate primitive -types. We aim to support heuristic-random, solver-based, and fuzzing-based backends. +Hypothesis supports alternative backends, which tells Hypothesis how to generate primitive +types. This enables powerful generation techniques which are compatible with all parts of +Hypothesis, including the database and shrinking. -See :issue:`3086` for details, e.g. if you're interested in writing your own backend. -(note that there is *no stable interface* for this; you'd be helping us work out -what that should eventually look like, and we're likely to make regular breaking -changes for some time to come) +Hypothesis includes the following backends: -Using the prototype :pypi:`crosshair-tool` backend via :pypi:`hypothesis-crosshair`, -a solver-backed test might look something like: +hypothesis + The default backend. +hypothesis-urandom + The same same as the default backend, but uses ``/dev/urandom`` to back the randomness + behind its PRNG. The only reason to use this backend is if you are also using + `Antithesis `_, in which case this enables Antithesis mutations + to drive Hypothesis generation. + + Not available on windows. +crosshair + Generates examples using SMT solvers like z3, which is particularly effective at satisfying + difficult checks in your code, like ``if`` or ``==`` statements. + + Requires ``pip install hypothesis[crosshair]``. + +You can change the backend for a test with the ``backend`` setting. For instance, after +``pip install hypothesis[crosshair]``, you can use :pypi:`crosshair ` to +generate examples with SMT via the :pypi:`hypothesis-crosshair` backend: .. code-block:: python from hypothesis import given, settings, strategies as st - @settings(backend="crosshair") # pip install hypothesis[crosshair] + # requires pip install hypothesis[crosshair] + @settings(backend="crosshair") @given(st.integers()) def test_needs_solver(x): assert x != 123456789 + +Failures found by alternative backends are saved to the database and shrink just like normally +generated examples, and in general interact with every feature of Hypothesis as you would expect. diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/providers.py b/hypothesis-python/src/hypothesis/internal/conjecture/providers.py index 8665a54b1f..08bed7cef5 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/providers.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/providers.py @@ -60,15 +60,19 @@ COLLECTION_DEFAULT_MAX_SIZE = 10**10 # "arbitrarily large" -# The set of available `PrimitiveProvider`s, by name. Other libraries, such as -# crosshair, can implement this interface and add themselves; at which point users -# can configure which backend to use via settings. Keys are the name of the library, -# which doubles as the backend= setting, and values are importable class names. +# The available `PrimitiveProvider`s, and therefore also the available backends +# for use by @settings(backend=...). The key is the name to be used in the backend= +# value, and the value is the importable path to a subclass of PrimitiveProvider. # -# NOTE: this is a temporary interface. We DO NOT promise to continue supporting it! -# (but if you want to experiment and don't mind breakage, here you go) +# See also +# https://hypothesis.readthedocs.io/en/latest/strategies.html#alternative-backends-for-hypothesis. +# +# NOTE: the PrimitiveProvider interface is not yet stable. We may continue to +# make breaking changes to it. (but if you want to experiment and don't mind +# breakage, here you go!) AVAILABLE_PROVIDERS = { "hypothesis": "hypothesis.internal.conjecture.providers.HypothesisProvider", + "hypothesis-urandom": "hypothesis.internal.conjecture.providers.URandomProvider", } FLOAT_INIT_LOGIC_CACHE = LRUCache(4096) STRING_SAMPLER_CACHE = LRUCache(64) diff --git a/hypothesis-python/tests/conjecture/test_alt_backend.py b/hypothesis-python/tests/conjecture/test_alt_backend.py index ffa629492d..ba91dcac7a 100644 --- a/hypothesis-python/tests/conjecture/test_alt_backend.py +++ b/hypothesis-python/tests/conjecture/test_alt_backend.py @@ -35,7 +35,7 @@ InvalidArgument, Unsatisfiable, ) -from hypothesis.internal.compat import int_to_bytes +from hypothesis.internal.compat import WINDOWS, int_to_bytes from hypothesis.internal.conjecture.data import ConjectureData, PrimitiveProvider from hypothesis.internal.conjecture.engine import ConjectureRunner from hypothesis.internal.conjecture.providers import ( @@ -596,3 +596,20 @@ def test_available_providers_deprecation(): with pytest.raises(ImportError): from hypothesis.internal.conjecture.data import does_not_exist # noqa + + +@pytest.mark.parametrize("backend", AVAILABLE_PROVIDERS.keys()) +@pytest.mark.parametrize( + "strategy", [st.integers(), st.text(), st.floats(), st.booleans(), st.binary()] +) +def test_can_generate_from_all_available_providers(backend, strategy): + if backend == "hypothesis-urandom" and WINDOWS: + pytest.skip("/dev/urandom not available on windows") + + @given(strategy) + @settings(backend=backend) + def f(x): + raise ValueError + + with pytest.raises(ValueError): + f() From 089e40f923ab2c7c6dacbb12d49c557679457dd4 Mon Sep 17 00:00:00 2001 From: Liam DeVoe Date: Mon, 10 Mar 2025 21:31:08 -0400 Subject: [PATCH 06/24] reword --- hypothesis-python/RELEASE.rst | 4 +++- hypothesis-python/docs/strategies.rst | 8 ++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst index 618a00cb32..7f4305304f 100644 --- a/hypothesis-python/RELEASE.rst +++ b/hypothesis-python/RELEASE.rst @@ -1,3 +1,5 @@ RELEASE_TYPE: minor -This patch adds an :ref:`alternative backend ` which draws randomness from ``/dev/urandom``. This is useful for users of `Antithesis `_ who also have Hypothesis tests, allowing Antithesis mutation of ``/dev/urandom`` to drive Hypothesis generation. +This release adds a ``"hypothesis-urandom"`` :ref:`backend `, which draws randomness from ``/dev/urandom`` instead of Python's PRNG. This is useful for users of `Antithesis `_ who also have Hypothesis tests, allowing Antithesis mutation of ``/dev/urandom`` to drive Hypothesis generation. We expect it to be strictly slower than the default backend for everyone else. + +It can be enabled with ``@settings(backend="hypothesis-urandom")``. diff --git a/hypothesis-python/docs/strategies.rst b/hypothesis-python/docs/strategies.rst index d3b0b81b52..82c8d7622d 100644 --- a/hypothesis-python/docs/strategies.rst +++ b/hypothesis-python/docs/strategies.rst @@ -253,10 +253,10 @@ Hypothesis includes the following backends: hypothesis The default backend. hypothesis-urandom - The same same as the default backend, but uses ``/dev/urandom`` to back the randomness - behind its PRNG. The only reason to use this backend is if you are also using - `Antithesis `_, in which case this enables Antithesis mutations - to drive Hypothesis generation. + The same as the default backend, but uses ``/dev/urandom`` to back the randomness + behind its PRNG. The only reason to use this backend over the default is if you are also + using `Antithesis `_, in which case this enables Antithesis + mutations to drive Hypothesis generation. Not available on windows. crosshair From 2333b5f9b58f3ce8241056933ee29ae5bcade968 Mon Sep 17 00:00:00 2001 From: Liam DeVoe Date: Tue, 11 Mar 2025 00:26:09 -0400 Subject: [PATCH 07/24] skip crosshair for now --- hypothesis-python/tests/conjecture/test_alt_backend.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/hypothesis-python/tests/conjecture/test_alt_backend.py b/hypothesis-python/tests/conjecture/test_alt_backend.py index ba91dcac7a..9fefd7477a 100644 --- a/hypothesis-python/tests/conjecture/test_alt_backend.py +++ b/hypothesis-python/tests/conjecture/test_alt_backend.py @@ -603,6 +603,14 @@ def test_available_providers_deprecation(): "strategy", [st.integers(), st.text(), st.floats(), st.booleans(), st.binary()] ) def test_can_generate_from_all_available_providers(backend, strategy): + if backend == "crosshair": + # TODO running into a 'not in statespace' issue which is fixed in + # https://github.com/HypothesisWorks/hypothesis/pull/4034. Remove + # this skip when that is merged" + pytest.skip( + "temp, fixed in https://github.com/HypothesisWorks/hypothesis/pull/4034" + ) + if backend == "hypothesis-urandom" and WINDOWS: pytest.skip("/dev/urandom not available on windows") From b339104a4f0ac60769309212ee618b241cef1fa7 Mon Sep 17 00:00:00 2001 From: Liam DeVoe Date: Tue, 11 Mar 2025 00:37:46 -0400 Subject: [PATCH 08/24] also implement random.random() --- .../src/hypothesis/internal/conjecture/providers.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/providers.py b/hypothesis-python/src/hypothesis/internal/conjecture/providers.py index 08bed7cef5..999437edea 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/providers.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/providers.py @@ -836,14 +836,22 @@ def draw_bytes( class URandom(Random): # we reimplement a Random instance instead of using SystemRandom, because # os.urandom is not guaranteed to read from /dev/urandom. + + def _urandom(self, size: int) -> bytes: + with open("/dev/urandom", "rb") as f: + return f.read(size) + def getrandbits(self, k: int) -> int: assert k >= 0 size = bits_to_bytes(k) - with open("/dev/urandom", "rb") as f: - n = int_from_bytes(bytearray(f.read(size))) + n = int_from_bytes(self._urandom(size)) # trim excess bits return n >> (size * 8 - k) + def random(self) -> float: + # adapted from random.SystemRandom.random + return (int_from_bytes(self._urandom(7)) >> 3) * (2**-53) + class URandomProvider(HypothesisProvider): # A provider which reads directly from /dev/urandom as its source of randomness. From c1a4d0361d79814c1d9b324ebd43a08969a7c053 Mon Sep 17 00:00:00 2001 From: CI on behalf of the Hypothesis team Date: Tue, 11 Mar 2025 17:46:21 +0000 Subject: [PATCH 09/24] Bump hypothesis-python version to 6.128.3 and update changelog [skip ci] --- hypothesis-python/RELEASE.rst | 22 ----------------- hypothesis-python/docs/changes.rst | 27 +++++++++++++++++++++ hypothesis-python/src/hypothesis/version.py | 2 +- 3 files changed, 28 insertions(+), 23 deletions(-) delete mode 100644 hypothesis-python/RELEASE.rst diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst deleted file mode 100644 index 7a735d9e3d..0000000000 --- a/hypothesis-python/RELEASE.rst +++ /dev/null @@ -1,22 +0,0 @@ -RELEASE_TYPE: patch - -For strategies which draw make recursive draws, including :func:`~hypothesis.strategies.recursive` and :func:`~hypothesis.strategies.deferred`, we now generate examples with duplicated subtrees more often. This tends to uncover interesting behavior in tests. - -For instance, we might now generate a tree like this more often (though the details depend on the strategy): - -.. code-block:: none - - ┌─────┐ - ┌──────┤ a ├──────┐ - │ └─────┘ │ - ┌──┴──┐ ┌──┴──┐ - │ b │ │ a │ - └──┬──┘ └──┬──┘ - ┌────┴────┐ ┌────┴────┐ - ┌──┴──┐ ┌──┴──┐ ┌──┴──┐ ┌──┴──┐ - │ c │ │ d │ │ b │ │ ... │ - └─────┘ └─────┘ └──┬──┘ └─────┘ - ┌────┴────┐ - ┌──┴──┐ ┌──┴──┐ - │ c │ │ d │ - └─────┘ └─────┘ diff --git a/hypothesis-python/docs/changes.rst b/hypothesis-python/docs/changes.rst index 6e889a24c5..fee565bdcc 100644 --- a/hypothesis-python/docs/changes.rst +++ b/hypothesis-python/docs/changes.rst @@ -18,6 +18,33 @@ Hypothesis 6.x .. include:: ../RELEASE.rst +.. _v6.128.3: + +-------------------- +6.128.3 - 2025-03-11 +-------------------- + +For strategies which draw make recursive draws, including :func:`~hypothesis.strategies.recursive` and :func:`~hypothesis.strategies.deferred`, we now generate examples with duplicated subtrees more often. This tends to uncover interesting behavior in tests. + +For instance, we might now generate a tree like this more often (though the details depend on the strategy): + +.. code-block:: none + + ┌─────┐ + ┌──────┤ a ├──────┐ + │ └─────┘ │ + ┌──┴──┐ ┌──┴──┐ + │ b │ │ a │ + └──┬──┘ └──┬──┘ + ┌────┴────┐ ┌────┴────┐ + ┌──┴──┐ ┌──┴──┐ ┌──┴──┐ ┌──┴──┐ + │ c │ │ d │ │ b │ │ ... │ + └─────┘ └─────┘ └──┬──┘ └─────┘ + ┌────┴────┐ + ┌──┴──┐ ┌──┴──┐ + │ c │ │ d │ + └─────┘ └─────┘ + .. _v6.128.2: -------------------- diff --git a/hypothesis-python/src/hypothesis/version.py b/hypothesis-python/src/hypothesis/version.py index 0beccde433..80d6b949c9 100644 --- a/hypothesis-python/src/hypothesis/version.py +++ b/hypothesis-python/src/hypothesis/version.py @@ -8,5 +8,5 @@ # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -__version_info__ = (6, 128, 2) +__version_info__ = (6, 128, 3) __version__ = ".".join(map(str, __version_info__)) From 7d8622b4f313d91d0326ba1ee11fd322b8eb83c0 Mon Sep 17 00:00:00 2001 From: Liam DeVoe Date: Tue, 11 Mar 2025 14:46:41 -0400 Subject: [PATCH 10/24] address review comments --- hypothesis-python/docs/strategies.rst | 3 +-- .../internal/conjecture/providers.py | 19 ++++++++++++++--- hypothesis-python/tests/common/arguments.py | 6 +++++- .../tests/conjecture/test_alt_backend.py | 21 ++++++++++++------- 4 files changed, 36 insertions(+), 13 deletions(-) diff --git a/hypothesis-python/docs/strategies.rst b/hypothesis-python/docs/strategies.rst index 82c8d7622d..cb6411c38f 100644 --- a/hypothesis-python/docs/strategies.rst +++ b/hypothesis-python/docs/strategies.rst @@ -274,8 +274,7 @@ generate examples with SMT via the :pypi:`hypothesis-crosshair` backend: from hypothesis import given, settings, strategies as st - # requires pip install hypothesis[crosshair] - @settings(backend="crosshair") + @settings(backend="crosshair") # pip install hypothesis[crosshair] @given(st.integers()) def test_needs_solver(x): assert x != 123456789 diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/providers.py b/hypothesis-python/src/hypothesis/internal/conjecture/providers.py index 999437edea..19922eb799 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/providers.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/providers.py @@ -11,6 +11,7 @@ import abc import contextlib import math +import warnings from collections.abc import Iterable from random import Random from sys import float_info @@ -25,8 +26,9 @@ Union, ) +from hypothesis.errors import HypothesisWarning from hypothesis.internal.cache import LRUCache -from hypothesis.internal.compat import int_from_bytes +from hypothesis.internal.compat import WINDOWS, int_from_bytes from hypothesis.internal.conjecture.choice import ( StringKWargs, choice_kwargs_key, @@ -837,7 +839,8 @@ class URandom(Random): # we reimplement a Random instance instead of using SystemRandom, because # os.urandom is not guaranteed to read from /dev/urandom. - def _urandom(self, size: int) -> bytes: + @staticmethod + def _urandom(size: int) -> bytes: with open("/dev/urandom", "rb") as f: return f.read(size) @@ -865,4 +868,14 @@ class URandomProvider(HypothesisProvider): def __init__(self, conjecturedata: Optional["ConjectureData"], /): super().__init__(conjecturedata) - self._random = URandom() + if WINDOWS: + warnings.warn( + "/dev/urandom is not available on windows. Falling back to " + 'standard PRNG generation (equivalent to backend="hypothesis").', + HypothesisWarning, + stacklevel=1, + ) + # don't overwrite the HypothesisProvider self._random attribute in + # this case + else: + self._random = URandom() diff --git a/hypothesis-python/tests/common/arguments.py b/hypothesis-python/tests/common/arguments.py index ea9385cf91..a47d007672 100644 --- a/hypothesis-python/tests/common/arguments.py +++ b/hypothesis-python/tests/common/arguments.py @@ -10,7 +10,7 @@ import pytest -from hypothesis import given +from hypothesis import given, settings from hypothesis.errors import InvalidArgument @@ -30,7 +30,11 @@ def argument_validation_test(bad_args): ("function", "args", "kwargs"), bad_args, ids=list(map(e_to_str, bad_args)) ) def test_raise_invalid_argument(function, args, kwargs): + # some invalid argument tests may find multiple distinct invalid inputs, + # which hypothesis raises as an exception group (and is not caught by + # pytest.raises). @given(function(*args, **kwargs)) + @settings(report_multiple_bugs=False) def test(x): pass diff --git a/hypothesis-python/tests/conjecture/test_alt_backend.py b/hypothesis-python/tests/conjecture/test_alt_backend.py index 9fefd7477a..d65fa02343 100644 --- a/hypothesis-python/tests/conjecture/test_alt_backend.py +++ b/hypothesis-python/tests/conjecture/test_alt_backend.py @@ -11,7 +11,7 @@ import itertools import math import sys -from contextlib import contextmanager +from contextlib import contextmanager, nullcontext from random import Random from typing import Optional @@ -32,6 +32,7 @@ BackendCannotProceed, Flaky, HypothesisException, + HypothesisWarning, InvalidArgument, Unsatisfiable, ) @@ -606,18 +607,24 @@ def test_can_generate_from_all_available_providers(backend, strategy): if backend == "crosshair": # TODO running into a 'not in statespace' issue which is fixed in # https://github.com/HypothesisWorks/hypothesis/pull/4034. Remove - # this skip when that is merged" + # this skip when that is merged pytest.skip( "temp, fixed in https://github.com/HypothesisWorks/hypothesis/pull/4034" ) - if backend == "hypothesis-urandom" and WINDOWS: - pytest.skip("/dev/urandom not available on windows") - @given(strategy) - @settings(backend=backend) + @settings(backend=backend, database=None) def f(x): raise ValueError - with pytest.raises(ValueError): + with ( + pytest.raises(ValueError), + ( + pytest.warns( + HypothesisWarning, match="/dev/urandom is not available on windows" + ) + if backend == "hypothesis-urandom" and WINDOWS + else nullcontext() + ), + ): f() From b1135dc8067c7cc4a17b071db6a164932b3b9112 Mon Sep 17 00:00:00 2001 From: Liam DeVoe Date: Tue, 11 Mar 2025 14:48:48 -0400 Subject: [PATCH 11/24] also update docs --- hypothesis-python/docs/strategies.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hypothesis-python/docs/strategies.rst b/hypothesis-python/docs/strategies.rst index cb6411c38f..b9963d6444 100644 --- a/hypothesis-python/docs/strategies.rst +++ b/hypothesis-python/docs/strategies.rst @@ -258,7 +258,8 @@ hypothesis-urandom using `Antithesis `_, in which case this enables Antithesis mutations to drive Hypothesis generation. - Not available on windows. + ``/dev/urandom`` is not available on Windows, so we emit a warning and fall back to the + hypothesis backend there. crosshair Generates examples using SMT solvers like z3, which is particularly effective at satisfying difficult checks in your code, like ``if`` or ``==`` statements. From ca1c977732f68c8c01f3af398043d022dabff653 Mon Sep 17 00:00:00 2001 From: Liam DeVoe Date: Tue, 11 Mar 2025 15:01:05 -0400 Subject: [PATCH 12/24] nocover windows branch --- .../src/hypothesis/internal/conjecture/providers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/providers.py b/hypothesis-python/src/hypothesis/internal/conjecture/providers.py index 19922eb799..62e0ec9b84 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/providers.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/providers.py @@ -868,7 +868,7 @@ class URandomProvider(HypothesisProvider): def __init__(self, conjecturedata: Optional["ConjectureData"], /): super().__init__(conjecturedata) - if WINDOWS: + if WINDOWS: # pragma: no branch warnings.warn( "/dev/urandom is not available on windows. Falling back to " 'standard PRNG generation (equivalent to backend="hypothesis").', From 8412689234eb3da80bd108df36b773399ed78f71 Mon Sep 17 00:00:00 2001 From: Liam DeVoe Date: Tue, 11 Mar 2025 15:22:53 -0400 Subject: [PATCH 13/24] I guess this needs nocover? --- .../src/hypothesis/internal/conjecture/providers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/providers.py b/hypothesis-python/src/hypothesis/internal/conjecture/providers.py index 62e0ec9b84..609b2dd687 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/providers.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/providers.py @@ -868,7 +868,7 @@ class URandomProvider(HypothesisProvider): def __init__(self, conjecturedata: Optional["ConjectureData"], /): super().__init__(conjecturedata) - if WINDOWS: # pragma: no branch + if WINDOWS: # pragma: no cover warnings.warn( "/dev/urandom is not available on windows. Falling back to " 'standard PRNG generation (equivalent to backend="hypothesis").', From a6b58c1c8e0fbf91766d56df9a05179441cefa14 Mon Sep 17 00:00:00 2001 From: CI on behalf of the Hypothesis team Date: Tue, 11 Mar 2025 21:31:48 +0000 Subject: [PATCH 14/24] Bump hypothesis-python version to 6.129.0 and update changelog [skip ci] --- hypothesis-python/RELEASE.rst | 5 ----- hypothesis-python/docs/changes.rst | 10 ++++++++++ hypothesis-python/src/hypothesis/version.py | 2 +- 3 files changed, 11 insertions(+), 6 deletions(-) delete mode 100644 hypothesis-python/RELEASE.rst diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst deleted file mode 100644 index 7f4305304f..0000000000 --- a/hypothesis-python/RELEASE.rst +++ /dev/null @@ -1,5 +0,0 @@ -RELEASE_TYPE: minor - -This release adds a ``"hypothesis-urandom"`` :ref:`backend `, which draws randomness from ``/dev/urandom`` instead of Python's PRNG. This is useful for users of `Antithesis `_ who also have Hypothesis tests, allowing Antithesis mutation of ``/dev/urandom`` to drive Hypothesis generation. We expect it to be strictly slower than the default backend for everyone else. - -It can be enabled with ``@settings(backend="hypothesis-urandom")``. diff --git a/hypothesis-python/docs/changes.rst b/hypothesis-python/docs/changes.rst index fee565bdcc..fffa10546f 100644 --- a/hypothesis-python/docs/changes.rst +++ b/hypothesis-python/docs/changes.rst @@ -18,6 +18,16 @@ Hypothesis 6.x .. include:: ../RELEASE.rst +.. _v6.129.0: + +-------------------- +6.129.0 - 2025-03-11 +-------------------- + +This release adds a ``"hypothesis-urandom"`` :ref:`backend `, which draws randomness from ``/dev/urandom`` instead of Python's PRNG. This is useful for users of `Antithesis `_ who also have Hypothesis tests, allowing Antithesis mutation of ``/dev/urandom`` to drive Hypothesis generation. We expect it to be strictly slower than the default backend for everyone else. + +It can be enabled with ``@settings(backend="hypothesis-urandom")``. + .. _v6.128.3: -------------------- diff --git a/hypothesis-python/src/hypothesis/version.py b/hypothesis-python/src/hypothesis/version.py index 80d6b949c9..b00a4fe5e6 100644 --- a/hypothesis-python/src/hypothesis/version.py +++ b/hypothesis-python/src/hypothesis/version.py @@ -8,5 +8,5 @@ # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -__version_info__ = (6, 128, 3) +__version_info__ = (6, 129, 0) __version__ = ".".join(map(str, __version_info__)) From cefe077086c7c64b284bc87c2b5e926cfe136fad Mon Sep 17 00:00:00 2001 From: Zac Hatfield-Dodds Date: Thu, 13 Mar 2025 02:38:48 -0700 Subject: [PATCH 15/24] tweak some old changelog entries --- hypothesis-python/docs/changes.rst | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/hypothesis-python/docs/changes.rst b/hypothesis-python/docs/changes.rst index fffa10546f..26fc111e25 100644 --- a/hypothesis-python/docs/changes.rst +++ b/hypothesis-python/docs/changes.rst @@ -317,7 +317,7 @@ Improves our internal caching logic for test cases. 6.124.1 - 2025-01-18 -------------------- -:ref:`fuzz_one_input ` is now implemented using an :ref:`alternative backend `. This brings the interpretation of the fuzzer-provided bytestring closer to the fuzzer mutations, allowing the mutations to work more reliably. We hope to use this backend functionality to improve fuzzing integration (see e.g. https://github.com/google/atheris/issues/20) in the future! +:ref:`fuzz_one_input ` is now implemented using an :ref:`alternative backend `. This brings the interpretation of the fuzzer-provided bytestring closer to the fuzzer mutations, allowing the mutations to work more reliably. We hope to use this backend functionality to improve fuzzing integration (e.g. `atheris issue #20 `__`) in the future! .. _v6.124.0: @@ -720,9 +720,7 @@ means that it's not possible for a server to actually be listening on port 0. This motivation is briefly described in the documentation for :func:`~hypothesis.provisional.urls`. -Fixes :issue:`4157`. - -Thanks to @gmacon for this contribution! +Thanks to @gmacon for fixing :issue:`4157`! .. _v6.117.0: @@ -928,7 +926,7 @@ in :func:`~hypothesis.strategies.from_type`. 6.110.1 - 2024-08-08 -------------------- -Add better error message for :obj:`!~python:typing.TypeIs` types +Add better error message for :obj:`~python:typing.TypeIs` types in :func:`~hypothesis.strategies.from_type`. .. _v6.110.0: @@ -1058,7 +1056,7 @@ which is used by the provisional :func:`~hypothesis.provisional.domains` strateg 6.108.0 - 2024-07-13 -------------------- -This patch changes most Flaky errors to use an ExceptionGroup, which +This patch changes most ``Flaky`` errors to use an :class:`ExceptionGroup`, which makes the representation of these errors easier to understand. .. _v6.107.0: @@ -1176,9 +1174,7 @@ Thanks to Joshua Munn for this contribution. -------------------- Fixes and reinstates full coverage of internal tests, which was accidentally -disabled in :pull:`3935`. - -Closes :issue:`4003`. +disabled in :pull:`3935` (:issue:`4003`). .. _v6.103.4: @@ -2773,9 +2769,7 @@ strategy. Thanks to Francesc Elies for :pull:`3602`. ------------------- This patch fixes a bug with :func:`~hypothesis.strategies.from_type()` with ``dict[tuple[int, int], str]`` -(:issue:`3527`). - - Thanks to Nick Muoh at the PyCon Sprints! +(:issue:`3527`). Thanks to Nick Muoh at the PyCon Sprints! .. _v6.72.2: @@ -3553,8 +3547,8 @@ methods, in addition to the existing support for functions and other callables 6.46.11 - 2022-06-02 -------------------- -Mention :func:`hypothesis.strategies.timezones` -in the documentation of :func:`hypothesis.strategies.datetimes` for completeness. +Mention :func:`~hypothesis.strategies.timezones` +in the documentation of :func:`~hypothesis.strategies.datetimes` for completeness. Thanks to George Macon for this addition. @@ -3805,7 +3799,7 @@ Now, when using with ``pytest-xdist``, the junit report will just omit the For more details, see `this pytest issue `__, `this pytest issue `__, -and :issue:`1935` +and :issue:`1935`. Thanks to Brandon Chinn for this bug fix! @@ -3824,7 +3818,7 @@ updates our vendored `list of top-level domains ` less noisy (:issue:`3253`), and generally improves pretty-printing of functions. .. _v6.41.0: @@ -3870,7 +3864,7 @@ Fixed :func:`~hypothesis.strategies.from_type` support for 6.40.1 - 2022-04-01 ------------------- -Fixed an internal error when ``given()`` was passed a lambda. +Fixed an internal error when :func:`~hypothesis.given` was passed a lambda. .. _v6.40.0: From 75ae68a3e1ddd79ea6daf690698492bbafb3d6ba Mon Sep 17 00:00:00 2001 From: Zac Hatfield-Dodds Date: Thu, 13 Mar 2025 02:38:48 -0700 Subject: [PATCH 16/24] fix bounds of ArtificialRandom --- hypothesis-python/RELEASE.rst | 4 ++++ hypothesis-python/docs/changes.rst | 2 +- .../hypothesis/strategies/_internal/random.py | 22 +++++++---------- hypothesis-python/tests/cover/test_randoms.py | 24 +++++++++++++++---- 4 files changed, 33 insertions(+), 19 deletions(-) create mode 100644 hypothesis-python/RELEASE.rst diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst new file mode 100644 index 0000000000..0f36c4b765 --- /dev/null +++ b/hypothesis-python/RELEASE.rst @@ -0,0 +1,4 @@ +RELEASE_TYPE: patch + +:func:`~hypothesis.strategies.randoms` no longer produces ``1.0``, matching +the exclusive upper bound of :obj:`random.Random.random` (:issue:`4297`). diff --git a/hypothesis-python/docs/changes.rst b/hypothesis-python/docs/changes.rst index 26fc111e25..307f64fa85 100644 --- a/hypothesis-python/docs/changes.rst +++ b/hypothesis-python/docs/changes.rst @@ -317,7 +317,7 @@ Improves our internal caching logic for test cases. 6.124.1 - 2025-01-18 -------------------- -:ref:`fuzz_one_input ` is now implemented using an :ref:`alternative backend `. This brings the interpretation of the fuzzer-provided bytestring closer to the fuzzer mutations, allowing the mutations to work more reliably. We hope to use this backend functionality to improve fuzzing integration (e.g. `atheris issue #20 `__`) in the future! +:ref:`fuzz_one_input ` is now implemented using an :ref:`alternative backend `. This brings the interpretation of the fuzzer-provided bytestring closer to the fuzzer mutations, allowing the mutations to work more reliably. We hope to use this backend functionality to improve fuzzing integration (e.g. `atheris issue #20 `__) in the future! .. _v6.124.0: diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/random.py b/hypothesis-python/src/hypothesis/strategies/_internal/random.py index 0e87459443..14acdf3366 100644 --- a/hypothesis-python/src/hypothesis/strategies/_internal/random.py +++ b/hypothesis-python/src/hypothesis/strategies/_internal/random.py @@ -18,12 +18,7 @@ from hypothesis.control import should_note from hypothesis.internal.reflection import define_function_signature from hypothesis.reporting import report -from hypothesis.strategies._internal.core import ( - binary, - lists, - permutations, - sampled_from, -) +from hypothesis.strategies._internal.core import lists, permutations, sampled_from from hypothesis.strategies._internal.numbers import floats, integers from hypothesis.strategies._internal.strategies import SearchStrategy @@ -171,9 +166,6 @@ def state_for_seed(data, seed): return state -UNIFORM = floats(0, 1) - - def normalize_zero(f: float) -> float: if f == 0.0: return 0.0 @@ -233,8 +225,12 @@ def _hypothesis_do_random(self, method, kwargs): if method == "_randbelow": result = self.__data.draw_integer(0, kwargs["n"] - 1) - elif method in ("betavariate", "random"): - result = self.__data.draw(UNIFORM) + elif method == "random": + # See https://github.com/HypothesisWorks/hypothesis/issues/4297 + # for numerics/bounds of "random" and "betavariate" + result = self.__data.draw(floats(0, 1, exclude_max=True)) + elif method == "betavariate": + result = self.__data.draw(floats(0, 1)) elif method == "uniform": a = normalize_zero(kwargs["a"]) b = normalize_zero(kwargs["b"]) @@ -324,8 +320,8 @@ def _hypothesis_do_random(self, method, kwargs): elif method == "shuffle": result = self.__data.draw(permutations(range(len(kwargs["x"])))) elif method == "randbytes": - n = kwargs["n"] - result = self.__data.draw(binary(min_size=n, max_size=n)) + n = int(kwargs["n"]) + result = self.__data.draw_bytes(min_size=n, max_size=n) else: raise NotImplementedError(method) diff --git a/hypothesis-python/tests/cover/test_randoms.py b/hypothesis-python/tests/cover/test_randoms.py index e9842a16a1..1cc2f900bb 100644 --- a/hypothesis-python/tests/cover/test_randoms.py +++ b/hypothesis-python/tests/cover/test_randoms.py @@ -25,7 +25,7 @@ normalize_zero, ) -from tests.common.debug import find_any +from tests.common.debug import assert_all_examples, find_any def test_implements_all_random_methods(): @@ -311,10 +311,6 @@ def test_samples_have_right_length(rnd, sample): assert len(rnd.sample(seq, k)) == k -@pytest.mark.skipif( - "choices" not in RANDOM_METHODS, - reason="choices not supported on this Python version", -) @given(st.randoms(use_true_random=False), any_call_of_method("choices")) def test_choices_have_right_length(rnd, choices): args, kwargs = choices @@ -384,3 +380,21 @@ def test_can_sample_from_large_subset(rnd): @given(st.randoms(use_true_random=False)) def test_can_draw_empty_from_empty_sequence(rnd): assert rnd.sample([], 0) == [] + + +def test_random_includes_zero_excludes_one(): + strat = st.randoms(use_true_random=False).map(lambda r: r.random()) + assert_all_examples(strat, lambda x: 0 <= x < 1) + find_any(strat, lambda x: x == 0) + + +def test_betavariate_includes_zero_and_one(): + # https://github.com/HypothesisWorks/hypothesis/issues/4297#issuecomment-2720144709 + strat = st.randoms(use_true_random=False).flatmap( + lambda r: st.builds( + r.betavariate, alpha=st.just(1.0) | beta_param, beta=beta_param + ) + ) + assert_all_examples(strat, lambda x: 0 <= x <= 1) + find_any(strat, lambda x: x == 0) + find_any(strat, lambda x: x == 1) From f62ec1e09ce787e6912c85d7ce6646dc17395116 Mon Sep 17 00:00:00 2001 From: CI on behalf of the Hypothesis team Date: Thu, 13 Mar 2025 18:07:16 +0000 Subject: [PATCH 17/24] Bump hypothesis-python version to 6.129.1 and update changelog [skip ci] --- hypothesis-python/RELEASE.rst | 4 ---- hypothesis-python/docs/changes.rst | 9 +++++++++ hypothesis-python/src/hypothesis/version.py | 2 +- 3 files changed, 10 insertions(+), 5 deletions(-) delete mode 100644 hypothesis-python/RELEASE.rst diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst deleted file mode 100644 index 0f36c4b765..0000000000 --- a/hypothesis-python/RELEASE.rst +++ /dev/null @@ -1,4 +0,0 @@ -RELEASE_TYPE: patch - -:func:`~hypothesis.strategies.randoms` no longer produces ``1.0``, matching -the exclusive upper bound of :obj:`random.Random.random` (:issue:`4297`). diff --git a/hypothesis-python/docs/changes.rst b/hypothesis-python/docs/changes.rst index 307f64fa85..e0987a58f0 100644 --- a/hypothesis-python/docs/changes.rst +++ b/hypothesis-python/docs/changes.rst @@ -18,6 +18,15 @@ Hypothesis 6.x .. include:: ../RELEASE.rst +.. _v6.129.1: + +-------------------- +6.129.1 - 2025-03-13 +-------------------- + +:func:`~hypothesis.strategies.randoms` no longer produces ``1.0``, matching +the exclusive upper bound of :obj:`random.Random.random` (:issue:`4297`). + .. _v6.129.0: -------------------- diff --git a/hypothesis-python/src/hypothesis/version.py b/hypothesis-python/src/hypothesis/version.py index b00a4fe5e6..ea5f816104 100644 --- a/hypothesis-python/src/hypothesis/version.py +++ b/hypothesis-python/src/hypothesis/version.py @@ -8,5 +8,5 @@ # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -__version_info__ = (6, 129, 0) +__version_info__ = (6, 129, 1) __version__ = ".".join(map(str, __version_info__)) From 9307192f2feeb18deec284a20415b1d175da1ef8 Mon Sep 17 00:00:00 2001 From: Liam DeVoe Date: Thu, 13 Mar 2025 18:22:08 -0400 Subject: [PATCH 18/24] use shrink key to avoid work in more places --- hypothesis-python/RELEASE.rst | 3 + .../internal/conjecture/shrinker.py | 76 +++++++++---------- 2 files changed, 39 insertions(+), 40 deletions(-) create mode 100644 hypothesis-python/RELEASE.rst diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst new file mode 100644 index 0000000000..1aff22db0e --- /dev/null +++ b/hypothesis-python/RELEASE.rst @@ -0,0 +1,3 @@ +RELEASE_TYPE: patch + +Improve how the shrinker checks for unnecessary work, leading to 10% less time spent shrinking on average, with no reduction in quality. diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/shrinker.py b/hypothesis-python/src/hypothesis/internal/conjecture/shrinker.py index 93cab7a519..5a6806c76d 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/shrinker.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/shrinker.py @@ -11,7 +11,7 @@ import math from collections import defaultdict from collections.abc import Sequence -from typing import TYPE_CHECKING, Callable, Optional, Union, cast +from typing import TYPE_CHECKING, Callable, Literal, Optional, Union, cast import attr @@ -136,7 +136,7 @@ class Shrinker: manage the associated state of a particular shrink problem. That is, we have some initial ConjectureData object and some property of interest that it satisfies, and we want to find a ConjectureData object with a - shortlex (see sort_key above) smaller buffer that exhibits the same + shortlex (see sort_key above) smaller choice sequence that exhibits the same property. Currently the only property of interest we use is that the status is @@ -160,7 +160,7 @@ class Shrinker: ======================= Generally a shrink pass is just any function that calls - cached_test_function and/or incorporate_new_buffer a number of times, + cached_test_function and/or consider_new_nodes a number of times, but there are a couple of useful things to bear in mind. A shrink pass *makes progress* if running it changes self.shrink_target @@ -202,22 +202,22 @@ class Shrinker: are carefully designed to do the right thing in the case that no shrinks occurred and try to adapt to any changes to do a reasonable job. e.g. say we wanted to write a shrink pass that tried deleting - each individual byte (this isn't an especially good choice, + each individual choice (this isn't an especially good pass, but it leads to a simple illustrative example), we might do it - by iterating over the buffer like so: + by iterating over the choice sequence like so: .. code-block:: python i = 0 - while i < len(self.shrink_target.buffer): - if not self.incorporate_new_buffer( - self.shrink_target.buffer[:i] + self.shrink_target.buffer[i + 1 :] + while i < len(self.shrink_target.nodes): + if not self.consider_new_nodes( + self.shrink_target.nodes[:i] + self.shrink_target.nodes[i + 1 :] ): i += 1 The reason for writing the loop this way is that i is always a - valid index into the current buffer, even if the current buffer - changes as a result of our actions. When the buffer changes, + valid index into the current choice sequence, even if the current sequence + changes as a result of our actions. When the choice sequence changes, we leave the index where it is rather than restarting from the beginning, and carry on. This means that the number of steps we run in this case is always bounded above by the number of steps @@ -308,10 +308,8 @@ def __init__( self.__predicate = predicate or (lambda data: True) self.__allow_transition = allow_transition or (lambda source, destination: True) self.__derived_values: dict = {} - self.__pending_shrink_explanation = None self.initial_size = len(initial.choices) - # We keep track of the current best example on the shrink_target # attribute. self.shrink_target = initial @@ -331,7 +329,7 @@ def __init__( # Because the shrinker is also used to `pareto_optimise` in the target phase, # we sometimes want to allow extending buffers instead of aborting at the end. - self.__extend = "full" if in_target_phase else 0 + self.__extend: Union[Literal["full"], int] = "full" if in_target_phase else 0 self.should_explain = explain @derived_value # type: ignore @@ -383,32 +381,32 @@ def check_calls(self) -> None: if self.calls - self.calls_at_last_shrink >= self.max_stall: raise StopShrinking - def cached_test_function(self, nodes): + def cached_test_function( + self, nodes: Sequence[ChoiceNode] + ) -> tuple[bool, Optional[Union[ConjectureResult, _Overrun]]]: + nodes = nodes[: len(self.nodes)] + + if startswith(nodes, self.nodes): + return (True, None) + + if sort_key(self.nodes) < sort_key(nodes): + return (False, None) + # sometimes our shrinking passes try obviously invalid things. We handle # discarding them in one place here. - for node in nodes: - if not choice_permitted(node.value, node.kwargs): - return None + if any(not choice_permitted(node.value, node.kwargs) for node in nodes): + return (False, None) result = self.engine.cached_test_function( [n.value for n in nodes], extend=self.__extend ) + previous = self.shrink_target self.incorporate_test_data(result) self.check_calls() - return result + return (previous is not self.shrink_target, result) def consider_new_nodes(self, nodes: Sequence[ChoiceNode]) -> bool: - nodes = nodes[: len(self.nodes)] - - if startswith(nodes, self.nodes): - return True - - if sort_key(self.nodes) < sort_key(nodes): - return False - - previous = self.shrink_target - self.cached_test_function(nodes) - return previous is not self.shrink_target + return self.cached_test_function(nodes)[0] def incorporate_test_data(self, data): """Takes a ConjectureData or Overrun object updates the current @@ -458,8 +456,8 @@ def s(n): "Shrink pass profiling\n" "---------------------\n\n" f"Shrinking made a total of {calls} call{s(calls)} of which " - f"{self.shrinks} shrank and {misaligned} were misaligned. This deleted {total_deleted} choices out " - f"of {self.initial_size}." + f"{self.shrinks} shrank and {misaligned} were misaligned. This " + f"deleted {total_deleted} choices out of {self.initial_size}." ) for useful in [True, False]: self.debug("") @@ -700,7 +698,7 @@ def reduce_each_alternative(self): # previous values to no longer be valid in its position. zero_attempt = self.cached_test_function( nodes[:i] + (nodes[i].copy(with_value=0),) + nodes[i + 1 :] - ) + )[1] if ( zero_attempt is not self.shrink_target and zero_attempt is not None @@ -731,10 +729,9 @@ def try_lower_node_as_alternative(self, i, v): while rerandomising and attempting to repair any subsequent changes to the shape of the test case that this causes.""" nodes = self.shrink_target.nodes - initial_attempt = self.cached_test_function( + if self.consider_new_nodes( nodes[:i] + (nodes[i].copy(with_value=v),) + nodes[i + 1 :] - ) - if initial_attempt is self.shrink_target: + ): return True prefix = nodes[:i] + (nodes[i].copy(with_value=v),) @@ -1090,7 +1087,7 @@ def try_shrinking_nodes(self, nodes, n): [(node.index, node.index + 1, [node.copy(with_value=n)]) for node in nodes], ) - attempt = self.cached_test_function(initial_attempt) + attempt = self.cached_test_function(initial_attempt)[1] if attempt is None: return False @@ -1149,8 +1146,7 @@ def try_shrinking_nodes(self, nodes, n): # attempts which increase min_size tend to overrun rather than # be misaligned, making a covering case difficult. return False # pragma: no cover - # the size decreased in our attempt. Try again, but replace with - # the min_size that we would have gotten, and truncate the value + # the size decreased in our attempt. Try again, but truncate the value # to that size by removing any elements past min_size. return self.consider_new_nodes( initial_attempt[: node.index] @@ -1534,7 +1530,7 @@ def try_trivial_spans(self, chooser): ] ) suffix = nodes[ex.end :] - attempt = self.cached_test_function(prefix + replacement + suffix) + attempt = self.cached_test_function(prefix + replacement + suffix)[1] if self.shrink_target is not prev: return @@ -1598,7 +1594,7 @@ def minimize_individual_choices(self, chooser): + (node.copy(with_value=node.value - 1),) + self.nodes[node.index + 1 :] ) - attempt = self.cached_test_function(lowered) + attempt = self.cached_test_function(lowered)[1] if ( attempt is None or attempt.status < Status.VALID From 282edb1a3376c2e452b2cb95409dfbe69b323304 Mon Sep 17 00:00:00 2001 From: Liam DeVoe Date: Fri, 14 Mar 2025 00:41:43 -0400 Subject: [PATCH 19/24] add covering case for newly missing coverage --- hypothesis-python/tests/conjecture/test_shrinker.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/hypothesis-python/tests/conjecture/test_shrinker.py b/hypothesis-python/tests/conjecture/test_shrinker.py index 615200606b..c5c14b9423 100644 --- a/hypothesis-python/tests/conjecture/test_shrinker.py +++ b/hypothesis-python/tests/conjecture/test_shrinker.py @@ -26,6 +26,7 @@ from hypothesis.internal.conjecture.utils import Sampler from hypothesis.internal.floats import MAX_PRECISE_INTEGER +from tests.common.debug import minimal from tests.conjecture.common import ( SOME_LABEL, float_kw, @@ -655,3 +656,15 @@ def shrinker(data: ConjectureData): shrinker.fixate_shrink_passes(["lower_duplicated_characters"]) assert shrinker.choices == (expected[0],) + (0,) * gap + (expected[1],) + + +def test_shrinking_one_of_with_same_shape(): + # This is a covering test for our one_of shrinking logic for the case when + # the choice sequence *doesn't* change shape in the newly chosen one_of branch. + # + # There are relatively few tests in our suite that cover this (and previously + # none in the covering subset). I chose the simplest one to copy here, but + # haven't yet put time into extracting the essence of a test case that + # covers this case, which is why we're using st.permutations here instead of + # something more fundamental / obviously testing what we want. + minimal(st.permutations(list(range(5))), lambda x: x[0] != 0) From c9c9464c1d1532142b742129c74f3ac425d5ca20 Mon Sep 17 00:00:00 2001 From: Liam DeVoe Date: Fri, 14 Mar 2025 19:02:42 -0400 Subject: [PATCH 20/24] improve one_of shrinking coverage case --- .../tests/conjecture/test_shrinker.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/hypothesis-python/tests/conjecture/test_shrinker.py b/hypothesis-python/tests/conjecture/test_shrinker.py index c5c14b9423..90f92f9150 100644 --- a/hypothesis-python/tests/conjecture/test_shrinker.py +++ b/hypothesis-python/tests/conjecture/test_shrinker.py @@ -26,7 +26,6 @@ from hypothesis.internal.conjecture.utils import Sampler from hypothesis.internal.floats import MAX_PRECISE_INTEGER -from tests.common.debug import minimal from tests.conjecture.common import ( SOME_LABEL, float_kw, @@ -661,10 +660,12 @@ def shrinker(data: ConjectureData): def test_shrinking_one_of_with_same_shape(): # This is a covering test for our one_of shrinking logic for the case when # the choice sequence *doesn't* change shape in the newly chosen one_of branch. - # - # There are relatively few tests in our suite that cover this (and previously - # none in the covering subset). I chose the simplest one to copy here, but - # haven't yet put time into extracting the essence of a test case that - # covers this case, which is why we're using st.permutations here instead of - # something more fundamental / obviously testing what we want. - minimal(st.permutations(list(range(5))), lambda x: x[0] != 0) + @shrinking_from([1, 0]) + def shrinker(data: ConjectureData): + n = data.draw_integer(0, 1) + data.draw_integer() + if n == 1: + data.mark_interesting() + + shrinker.initial_coarse_reduction() + assert shrinker.choices == (1, 0) From d01737e106cfef4cdb4af2d1f1630b3f25093a65 Mon Sep 17 00:00:00 2001 From: CI on behalf of the Hypothesis team Date: Fri, 14 Mar 2025 23:36:12 +0000 Subject: [PATCH 21/24] Bump hypothesis-python version to 6.129.2 and update changelog [skip ci] --- hypothesis-python/RELEASE.rst | 3 --- hypothesis-python/docs/changes.rst | 8 ++++++++ hypothesis-python/src/hypothesis/version.py | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) delete mode 100644 hypothesis-python/RELEASE.rst diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst deleted file mode 100644 index 1aff22db0e..0000000000 --- a/hypothesis-python/RELEASE.rst +++ /dev/null @@ -1,3 +0,0 @@ -RELEASE_TYPE: patch - -Improve how the shrinker checks for unnecessary work, leading to 10% less time spent shrinking on average, with no reduction in quality. diff --git a/hypothesis-python/docs/changes.rst b/hypothesis-python/docs/changes.rst index e0987a58f0..940fca6143 100644 --- a/hypothesis-python/docs/changes.rst +++ b/hypothesis-python/docs/changes.rst @@ -18,6 +18,14 @@ Hypothesis 6.x .. include:: ../RELEASE.rst +.. _v6.129.2: + +-------------------- +6.129.2 - 2025-03-14 +-------------------- + +Improve how the shrinker checks for unnecessary work, leading to 10% less time spent shrinking on average, with no reduction in quality. + .. _v6.129.1: -------------------- diff --git a/hypothesis-python/src/hypothesis/version.py b/hypothesis-python/src/hypothesis/version.py index ea5f816104..fcf7fd036c 100644 --- a/hypothesis-python/src/hypothesis/version.py +++ b/hypothesis-python/src/hypothesis/version.py @@ -8,5 +8,5 @@ # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -__version_info__ = (6, 129, 1) +__version_info__ = (6, 129, 2) __version__ = ".".join(map(str, __version_info__)) From 0b2cfc0c47c8f86775a33fdd62ace76b9e1a4551 Mon Sep 17 00:00:00 2001 From: Liam DeVoe Date: Sat, 15 Mar 2025 18:45:17 -0400 Subject: [PATCH 22/24] merge fixed_dictionaries strategies --- hypothesis-python/RELEASE.rst | 3 + .../strategies/_internal/collections.py | 64 ++++++++----------- .../hypothesis/strategies/_internal/core.py | 8 +-- 3 files changed, 34 insertions(+), 41 deletions(-) create mode 100644 hypothesis-python/RELEASE.rst diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst new file mode 100644 index 0000000000..049868120b --- /dev/null +++ b/hypothesis-python/RELEASE.rst @@ -0,0 +1,3 @@ +RELEASE_TYPE: patch + +This patch improves the string representation of :func:`~hypothesis.strategies.fixed_dictionaries`. diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/collections.py b/hypothesis-python/src/hypothesis/strategies/_internal/collections.py index e647ec07fb..2b77439c24 100644 --- a/hypothesis-python/src/hypothesis/strategies/_internal/collections.py +++ b/hypothesis-python/src/hypothesis/strategies/_internal/collections.py @@ -10,8 +10,9 @@ import copy from collections.abc import Iterable -from typing import Any, overload +from typing import Any, Optional, overload +from hypothesis import strategies as st from hypothesis.errors import InvalidArgument from hypothesis.internal.conjecture import utils as cu from hypothesis.internal.conjecture.engine import BUFFER_SIZE @@ -24,7 +25,6 @@ T4, T5, Ex, - MappedStrategy, SearchStrategy, T, check_strategy, @@ -306,7 +306,7 @@ def do_draw(self, data): return result -class FixedKeysDictStrategy(MappedStrategy): +class FixedDictStrategy(SearchStrategy): """A strategy which produces dicts with a fixed set of keys, given a strategy for each of their equivalent values. @@ -314,42 +314,24 @@ class FixedKeysDictStrategy(MappedStrategy): key 'foo' mapping to some integer. """ - def __init__(self, strategy_dict): - dict_type = type(strategy_dict) - self.keys = tuple(strategy_dict.keys()) - super().__init__( - strategy=TupleStrategy(strategy_dict[k] for k in self.keys), - pack=lambda value: dict_type(zip(self.keys, value)), + def __init__( + self, + mapping: dict[T, SearchStrategy[Ex]], + *, + optional: Optional[dict[T, SearchStrategy[Ex]]], + ): + dict_type = type(mapping) + keys = tuple(mapping.keys()) + self.fixed = st.tuples(*[mapping[k] for k in keys]).map( + lambda value: dict_type(zip(keys, value)) ) - - def calc_is_empty(self, recur): - return recur(self.mapped_strategy) - - def __repr__(self): - return f"FixedKeysDictStrategy({self.keys!r}, {self.mapped_strategy!r})" - - -class FixedAndOptionalKeysDictStrategy(SearchStrategy): - """A strategy which produces dicts with a fixed set of keys, given a - strategy for each of their equivalent values. - - e.g. {'foo' : some_int_strategy} would generate dicts with the single - key 'foo' mapping to some integer. - """ - - def __init__(self, strategy_dict, optional): - self.required = strategy_dict - self.fixed = FixedKeysDictStrategy(strategy_dict) self.optional = optional - def calc_is_empty(self, recur): - return recur(self.fixed) - - def __repr__(self): - return f"FixedAndOptionalKeysDictStrategy({self.required!r}, {self.optional!r})" - def do_draw(self, data): - result = data.draw(self.fixed) + value = data.draw(self.fixed) + if self.optional is None: + return value + remaining = [k for k, v in self.optional.items() if not v.is_empty] should_draw = cu.many( data, min_size=0, max_size=len(remaining), average_size=len(remaining) / 2 @@ -358,5 +340,13 @@ def do_draw(self, data): j = data.draw_integer(0, len(remaining) - 1) remaining[-1], remaining[j] = remaining[j], remaining[-1] key = remaining.pop() - result[key] = data.draw(self.optional[key]) - return result + value[key] = data.draw(self.optional[key]) + return value + + def calc_is_empty(self, recur): + return recur(self.fixed) + + def __repr__(self): + if self.optional is not None: + return f"fixed_dictionaries({self.keys!r}, optional={self.optional!r})" + return f"fixed_dictionaries({self.keys!r})" diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/core.py b/hypothesis-python/src/hypothesis/strategies/_internal/core.py index 9a523a1fea..8f96f77b06 100644 --- a/hypothesis-python/src/hypothesis/strategies/_internal/core.py +++ b/hypothesis-python/src/hypothesis/strategies/_internal/core.py @@ -105,8 +105,7 @@ ) from hypothesis.strategies._internal import SearchStrategy, check_strategy from hypothesis.strategies._internal.collections import ( - FixedAndOptionalKeysDictStrategy, - FixedKeysDictStrategy, + FixedDictStrategy, ListStrategy, TupleStrategy, UniqueListStrategy, @@ -510,6 +509,7 @@ def fixed_dictionaries( check_type(dict, mapping, "mapping") for k, v in mapping.items(): check_strategy(v, f"mapping[{k!r}]") + if optional is not None: check_type(dict, optional, "optional") for k, v in optional.items(): @@ -524,8 +524,8 @@ def fixed_dictionaries( "The following keys were in both mapping and optional, " f"which is invalid: {set(mapping) & set(optional)!r}" ) - return FixedAndOptionalKeysDictStrategy(mapping, optional) - return FixedKeysDictStrategy(mapping) + + return FixedDictStrategy(mapping, optional=optional) @cacheable From c03724b3313142bd59b75bc65cd9531f0e054eaf Mon Sep 17 00:00:00 2001 From: Liam DeVoe Date: Sat, 15 Mar 2025 20:35:43 -0400 Subject: [PATCH 23/24] fix bad attribute access --- .../src/hypothesis/strategies/_internal/collections.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/collections.py b/hypothesis-python/src/hypothesis/strategies/_internal/collections.py index 2b77439c24..489f8bb604 100644 --- a/hypothesis-python/src/hypothesis/strategies/_internal/collections.py +++ b/hypothesis-python/src/hypothesis/strategies/_internal/collections.py @@ -321,6 +321,7 @@ def __init__( optional: Optional[dict[T, SearchStrategy[Ex]]], ): dict_type = type(mapping) + self.mapping = mapping keys = tuple(mapping.keys()) self.fixed = st.tuples(*[mapping[k] for k in keys]).map( lambda value: dict_type(zip(keys, value)) @@ -348,5 +349,5 @@ def calc_is_empty(self, recur): def __repr__(self): if self.optional is not None: - return f"fixed_dictionaries({self.keys!r}, optional={self.optional!r})" - return f"fixed_dictionaries({self.keys!r})" + return f"fixed_dictionaries({self.mapping!r}, optional={self.optional!r})" + return f"fixed_dictionaries({self.mapping!r})" From 2c0038f9ceb24f13fef51c5c06675b9093be3733 Mon Sep 17 00:00:00 2001 From: CI on behalf of the Hypothesis team Date: Sun, 16 Mar 2025 08:19:31 +0000 Subject: [PATCH 24/24] Bump hypothesis-python version to 6.129.3 and update changelog [skip ci] --- hypothesis-python/RELEASE.rst | 3 --- hypothesis-python/docs/changes.rst | 8 ++++++++ hypothesis-python/src/hypothesis/version.py | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) delete mode 100644 hypothesis-python/RELEASE.rst diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst deleted file mode 100644 index 049868120b..0000000000 --- a/hypothesis-python/RELEASE.rst +++ /dev/null @@ -1,3 +0,0 @@ -RELEASE_TYPE: patch - -This patch improves the string representation of :func:`~hypothesis.strategies.fixed_dictionaries`. diff --git a/hypothesis-python/docs/changes.rst b/hypothesis-python/docs/changes.rst index 940fca6143..8599642a68 100644 --- a/hypothesis-python/docs/changes.rst +++ b/hypothesis-python/docs/changes.rst @@ -18,6 +18,14 @@ Hypothesis 6.x .. include:: ../RELEASE.rst +.. _v6.129.3: + +-------------------- +6.129.3 - 2025-03-16 +-------------------- + +This patch improves the string representation of :func:`~hypothesis.strategies.fixed_dictionaries`. + .. _v6.129.2: -------------------- diff --git a/hypothesis-python/src/hypothesis/version.py b/hypothesis-python/src/hypothesis/version.py index fcf7fd036c..5e45daacda 100644 --- a/hypothesis-python/src/hypothesis/version.py +++ b/hypothesis-python/src/hypothesis/version.py @@ -8,5 +8,5 @@ # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -__version_info__ = (6, 129, 2) +__version_info__ = (6, 129, 3) __version__ = ".".join(map(str, __version_info__))