From e5b92a0247ec54590796512c8f90868a194eef04 Mon Sep 17 00:00:00 2001 From: Zac Hatfield-Dodds Date: Tue, 30 Jan 2024 13:49:06 +1100 Subject: [PATCH 1/3] Uncap buffer size in reuse phase --- hypothesis-python/RELEASE.rst | 10 ++++++++++ hypothesis-python/src/hypothesis/core.py | 3 +-- .../src/hypothesis/internal/conjecture/engine.py | 4 ++-- 3 files changed, 13 insertions(+), 4 deletions(-) create mode 100644 hypothesis-python/RELEASE.rst diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst new file mode 100644 index 0000000000..47f5bd2262 --- /dev/null +++ b/hypothesis-python/RELEASE.rst @@ -0,0 +1,10 @@ +RELEASE_TYPE: patch + +This patch slightly changes how we replay examples from +:doc:`the database `: if the behavior of the saved example has +changed, we now keep running the test case instead of aborting at the size +of the saved example. While we know it's not the *same* example, we might +as well continue running the test! + +Because we now finish running a few more examples for affected tests, this +might be a slight slowdown - but correspondingly more likely to find a bug. diff --git a/hypothesis-python/src/hypothesis/core.py b/hypothesis-python/src/hypothesis/core.py index 17876c3715..b96650d073 100644 --- a/hypothesis-python/src/hypothesis/core.py +++ b/hypothesis-python/src/hypothesis/core.py @@ -1510,8 +1510,7 @@ def wrapped_test(*arguments, **kwargs): except UnsatisfiedAssumption: raise DidNotReproduce( "The test data failed to satisfy an assumption in the " - "test. Have you added it since this blob was " - "generated?" + "test. Have you added it since this blob was generated?" ) from None # There was no @reproduce_failure, so start by running any explicit diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/engine.py b/hypothesis-python/src/hypothesis/internal/conjecture/engine.py index 961774816f..953b8842df 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/engine.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/engine.py @@ -554,7 +554,7 @@ def reuse_existing_examples(self): corpus.extend(extra) for existing in corpus: - data = self.cached_test_function(existing) + data = self.cached_test_function(existing, extend=BUFFER_SIZE) if data.status != Status.INTERESTING: self.settings.database.delete(self.database_key, existing) self.settings.database.delete(self.secondary_key, existing) @@ -569,7 +569,7 @@ def reuse_existing_examples(self): pareto_corpus.sort(key=sort_key) for existing in pareto_corpus: - data = self.cached_test_function(existing) + data = self.cached_test_function(existing, extend=BUFFER_SIZE) if data not in self.pareto_front: self.settings.database.delete(self.pareto_key, existing) if data.status == Status.INTERESTING: From 958f449187b535feb3334f5b2f2e493ce713c039 Mon Sep 17 00:00:00 2001 From: Zac Hatfield-Dodds Date: Tue, 30 Jan 2024 13:49:06 +1100 Subject: [PATCH 2/3] Observe phase information --- hypothesis-python/src/hypothesis/core.py | 4 +++- .../src/hypothesis/internal/conjecture/engine.py | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/hypothesis-python/src/hypothesis/core.py b/hypothesis-python/src/hypothesis/core.py index b96650d073..83bcaf049b 100644 --- a/hypothesis-python/src/hypothesis/core.py +++ b/hypothesis-python/src/hypothesis/core.py @@ -1054,6 +1054,8 @@ def _execute_once_for_engine(self, data: ConjectureData) -> None: if TESTCASE_CALLBACKS: if self.failed_normally or self.failed_due_to_deadline: phase = "shrink" + elif runner := getattr(self, "_runner", None): + phase = runner._current_phase else: phase = "unknown" tc = make_testcase( @@ -1084,7 +1086,7 @@ def run_engine(self): else: database_key = None - runner = ConjectureRunner( + runner = self._runner = ConjectureRunner( self._execute_once_for_engine, settings=self.settings, random=self.random, diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/engine.py b/hypothesis-python/src/hypothesis/internal/conjecture/engine.py index 953b8842df..a7b9ac6aff 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/engine.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/engine.py @@ -175,6 +175,7 @@ def _log_phase_statistics(self, phase): self.stats_per_test_case.clear() start_time = time.perf_counter() try: + self._current_phase = phase yield finally: self.statistics[phase + "-phase"] = { @@ -693,6 +694,7 @@ def generate_new_examples(self): ran_optimisations = False while self.should_generate_more(): + self._current_phase = "generate" prefix = self.generate_novel_prefix() assert len(prefix) <= BUFFER_SIZE if ( @@ -763,6 +765,7 @@ def generate_new_examples(self): and not ran_optimisations ): ran_optimisations = True + self._current_phase = "target" self.optimise_targets() def generate_mutations_from(self, data): @@ -902,6 +905,7 @@ def _run(self): # but if we've been asked to run it but not generation then we have to # run it explciitly on its own here. if Phase.generate not in self.settings.phases: + self._current_phase = "target" self.optimise_targets() with self._log_phase_statistics("shrink"): self.shrink_interesting_examples() From 91c63bb76c970effd6cf3c013d8ed98788cf0527 Mon Sep 17 00:00:00 2001 From: Zac Hatfield-Dodds Date: Tue, 30 Jan 2024 13:49:06 +1100 Subject: [PATCH 3/3] faster target phase --- hypothesis-python/RELEASE.rst | 3 +++ hypothesis-python/src/hypothesis/core.py | 2 +- .../src/hypothesis/internal/conjecture/engine.py | 5 ++++- .../src/hypothesis/internal/conjecture/pareto.py | 2 +- .../src/hypothesis/internal/conjecture/shrinker.py | 11 ++++++++++- 5 files changed, 19 insertions(+), 4 deletions(-) diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst index 47f5bd2262..eba792fdc2 100644 --- a/hypothesis-python/RELEASE.rst +++ b/hypothesis-python/RELEASE.rst @@ -8,3 +8,6 @@ as well continue running the test! Because we now finish running a few more examples for affected tests, this might be a slight slowdown - but correspondingly more likely to find a bug. + +We've also applied similar tricks to the :ref:`target phase `, where +they are a pure performance improvement for affected tests. diff --git a/hypothesis-python/src/hypothesis/core.py b/hypothesis-python/src/hypothesis/core.py index 83bcaf049b..73b51a316f 100644 --- a/hypothesis-python/src/hypothesis/core.py +++ b/hypothesis-python/src/hypothesis/core.py @@ -1056,7 +1056,7 @@ def _execute_once_for_engine(self, data: ConjectureData) -> None: phase = "shrink" elif runner := getattr(self, "_runner", None): phase = runner._current_phase - else: + else: # pragma: no cover # in case of messing with internals phase = "unknown" tc = make_testcase( start_timestamp=self._start_timestamp, diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/engine.py b/hypothesis-python/src/hypothesis/internal/conjecture/engine.py index a7b9ac6aff..99a170ca64 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/engine.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/engine.py @@ -129,6 +129,7 @@ def __init__( # Global dict of per-phase statistics, and a list of per-call stats # which transfer to the global dict at the end of each phase. + self._current_phase = "(not a phase)" self.statistics = {} self.stats_per_test_case = [] @@ -887,7 +888,8 @@ def optimise_targets(self): if any_improvements: continue - self.pareto_optimise() + if self.best_observed_targets: + self.pareto_optimise() if prev_calls == self.call_count: break @@ -1015,6 +1017,7 @@ def new_shrinker(self, example, predicate=None, allow_transition=None): predicate, allow_transition=allow_transition, explain=Phase.explain in self.settings.phases, + in_target_phase=self._current_phase == "target", ) def cached_test_function(self, buffer, *, error_on_discard=False, extend=0): diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/pareto.py b/hypothesis-python/src/hypothesis/internal/conjecture/pareto.py index d82408f97e..146b1b56f4 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/pareto.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/pareto.py @@ -317,7 +317,7 @@ def allow_transition(source, destination): # If ``destination`` dominates ``source`` then ``source`` # must be dominated in the front - either ``destination`` is in # the front, or it was not added to it because it was - # dominated by something in it., + # dominated by something in it. try: self.front.front.remove(source) except ValueError: diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/shrinker.py b/hypothesis-python/src/hypothesis/internal/conjecture/shrinker.py index f965829759..39a515d296 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/shrinker.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/shrinker.py @@ -271,6 +271,7 @@ def __init__( *, allow_transition: bool, explain: bool, + in_target_phase: bool = False, ): """Create a shrinker for a particular engine, with a given starting point and predicate. When shrink() is called it will attempt to find an @@ -309,6 +310,14 @@ def __init__( # testing and learning purposes. self.extra_dfas: Dict[str, ConcreteDFA] = {} + # Because the shrinker is also used to `pareto_optimise` in the target phase, + # we sometimes want to allow extending buffers instead of aborting at the end. + if in_target_phase: + from hypothesis.internal.conjecture.engine import BUFFER_SIZE + + self.__extend = BUFFER_SIZE + else: + self.__extend = 0 self.should_explain = explain @derived_value # type: ignore @@ -417,7 +426,7 @@ def cached_test_function(self, buffer): with status >= INVALID that would result from running this buffer.""" buffer = bytes(buffer) - result = self.engine.cached_test_function(buffer) + result = self.engine.cached_test_function(buffer, extend=self.__extend) self.incorporate_test_data(result) if self.calls - self.calls_at_last_shrink >= self.max_stall: raise StopShrinking