Add initial property-based tests using Hypothesis (#22280)

Zac-HD · TomAugspurger · commit fa47b8d95e47 · 2018-08-24T21:05:40.000-05:00
* BLD: Add Hypothesis to build system

* TST: Add Hypothesis tests for ticks, offsets

These tests are derived from GH18761, by jbrockmendel

Co-authored-by: jbrockmendel &lt;jbrockmendel@users.noreply.github.com&gt;

* DOC: Explain Hypothesis in contributing guide

* TST: remove pointless loop

* TST: Improve integration of Hypothesis

Responding to review from jreback on GH22280.

* Final review fixes
diff --git a/.gitignore b/.gitignore
@@ -62,6 +62,8 @@ dist
 coverage.xml
 coverage_html_report
 *.pytest_cache
+# hypothesis test database
+.hypothesis/
 
 # OS generated files #
 ######################
diff --git a/ci/appveyor-27.yaml b/ci/appveyor-27.yaml
@@ -28,3 +28,4 @@ dependencies:
   - pytest
   - pytest-xdist
   - moto
+  - hypothesis>=3.58.0
diff --git a/ci/appveyor-36.yaml b/ci/appveyor-36.yaml
@@ -25,3 +25,4 @@ dependencies:
   - cython>=0.28.2
   - pytest
   - pytest-xdist
+  - hypothesis>=3.58.0
diff --git a/ci/check_imports.py b/ci/check_imports.py
@@ -9,6 +9,7 @@
     'html5lib',
     'ipython',
     'jinja2'
+    'hypothesis',
     'lxml',
     'numexpr',
     'openpyxl',
diff --git a/ci/circle-27-compat.yaml b/ci/circle-27-compat.yaml
@@ -26,3 +26,4 @@ dependencies:
     - html5lib==1.0b2
     - beautifulsoup4==4.2.1
     - pymysql==0.6.0
+    - hypothesis>=3.58.0
diff --git a/ci/circle-35-ascii.yaml b/ci/circle-35-ascii.yaml
@@ -11,3 +11,5 @@ dependencies:
   # universal
   - pytest
   - pytest-xdist
+  - pip:
+    - hypothesis>=3.58.0
diff --git a/ci/circle-36-locale.yaml b/ci/circle-36-locale.yaml
@@ -31,3 +31,5 @@ dependencies:
   - pytest
   - pytest-xdist
   - moto
+  - pip:
+    - hypothesis>=3.58.0
diff --git a/ci/circle-36-locale_slow.yaml b/ci/circle-36-locale_slow.yaml
@@ -32,3 +32,5 @@ dependencies:
   - pytest
   - pytest-xdist
   - moto
+  - pip:
+    - hypothesis>=3.58.0
diff --git a/ci/environment-dev.yaml b/ci/environment-dev.yaml
@@ -7,6 +7,7 @@ dependencies:
   - NumPy
   - flake8
   - flake8-comprehensions
+  - hypothesis>=3.58.0
   - moto
   - pytest>=3.6
   - python-dateutil>=2.5.0
diff --git a/ci/requirements_dev.txt b/ci/requirements_dev.txt
@@ -4,6 +4,7 @@ Cython>=0.28.2
 NumPy
 flake8
 flake8-comprehensions
+hypothesis>=3.58.0
 moto
 pytest>=3.6
 python-dateutil>=2.5.0
diff --git a/ci/travis-27-locale.yaml b/ci/travis-27-locale.yaml
@@ -22,6 +22,7 @@ dependencies:
   # universal
   - pytest
   - pytest-xdist
+  - hypothesis>=3.58.0
   - pip:
     - html5lib==1.0b2
     - beautifulsoup4==4.2.1
diff --git a/ci/travis-27.yaml b/ci/travis-27.yaml
@@ -45,6 +45,7 @@ dependencies:
   - pytest
   - pytest-xdist
   - moto
+  - hypothesis>=3.58.0
   - pip:
     - backports.lzma
     - cpplint
diff --git a/ci/travis-35-osx.yaml b/ci/travis-35-osx.yaml
@@ -25,3 +25,4 @@ dependencies:
   - pytest-xdist
   - pip:
     - python-dateutil==2.5.3
+    - hypothesis>=3.58.0
diff --git a/ci/travis-36-doc.yaml b/ci/travis-36-doc.yaml
@@ -10,6 +10,7 @@ dependencies:
   - fastparquet
   - feather-format
   - html5lib
+  - hypothesis>=3.58.0
   - ipykernel
   - ipython
   - ipywidgets
diff --git a/ci/travis-36-numpydev.yaml b/ci/travis-36-numpydev.yaml
@@ -8,6 +8,7 @@ dependencies:
   # universal
   - pytest
   - pytest-xdist
+  - hypothesis>=3.58.0
   - pip:
     - "git+git://github.com/dateutil/dateutil.git"
     - "-f https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com"
diff --git a/ci/travis-36-slow.yaml b/ci/travis-36-slow.yaml
@@ -28,3 +28,4 @@ dependencies:
   - pytest
   - pytest-xdist
   - moto
+  - hypothesis>=3.58.0
diff --git a/ci/travis-36.yaml b/ci/travis-36.yaml
@@ -41,6 +41,7 @@ dependencies:
   - pytest-xdist
   - pytest-cov
   - moto
+  - hypothesis>=3.58.0
   - pip:
     - brotlipy
     - coverage
diff --git a/ci/travis-37.yaml b/ci/travis-37.yaml
@@ -12,3 +12,4 @@ dependencies:
   - pytz
   - pytest
   - pytest-xdist
+  - hypothesis>=3.58.0
diff --git a/doc/source/contributing.rst b/doc/source/contributing.rst
@@ -820,6 +820,46 @@ Tests that we have ``parametrized`` are now accessible via the test name, for ex
    test_cool_feature.py::test_series[int8] PASSED
 
 
+.. _using-hypothesis:
+
+Using ``hypothesis``
+~~~~~~~~~~~~~~~~~~~~
+
+Hypothesis is a library for property-based testing.  Instead of explicitly
+parametrizing a test, you can describe *all* valid inputs and let Hypothesis
+try to find a failing input.  Even better, no matter how many random examples
+it tries, Hypothesis always reports a single minimal counterexample to your
+assertions - often an example that you would never have thought to test.
+
+See `Getting Started with Hypothesis <https://hypothesis.works/articles/getting-started-with-hypothesis/>`_
+for more of an introduction, then `refer to the Hypothesis documentation
+for details <https://hypothesis.readthedocs.io/en/latest/index.html>`_.
+
+.. code-block:: python
+
+    import json
+    from hypothesis import given, strategies as st
+
+    any_json_value = st.deferred(lambda: st.one_of(
+        st.none(), st.booleans(), st.floats(allow_nan=False), st.text(),
+        st.lists(any_json_value), st.dictionaries(st.text(), any_json_value)
+    ))
+
+    @given(value=any_json_value)
+    def test_json_roundtrip(value):
+        result = json.loads(json.dumps(value))
+        assert value == result
+
+This test shows off several useful features of Hypothesis, as well as
+demonstrating a good use-case: checking properties that should hold over
+a large or complicated domain of inputs.
+
+To keep the Pandas test suite running quickly, parametrized tests are
+preferred if the inputs or logic are simple, with Hypothesis tests reserved
+for cases with complex logic or where there are too many combinations of
+options or subtle interactions to test (or think of!) all of them.
+
+
 Running the test suite
 ----------------------
 
diff --git a/doc/source/install.rst b/doc/source/install.rst
@@ -202,7 +202,8 @@ pandas is equipped with an exhaustive set of unit tests, covering about 97% of
 the code base as of this writing. To run it on your machine to verify that
 everything is working (and that you have all of the dependencies, soft and hard,
 installed), make sure you have `pytest
-<http://docs.pytest.org/en/latest/>`__ >= 3.6 and run:
+<http://docs.pytest.org/en/latest/>`__ >= 3.6 and `Hypothesis
+<https://hypothesis.readthedocs.io/>`__ >= 3.58, then run:
 
 ::
 
diff --git a/doc/source/whatsnew/v0.24.0.txt b/doc/source/whatsnew/v0.24.0.txt
@@ -729,6 +729,7 @@ Build Changes
 ^^^^^^^^^^^^^
 
 - Building pandas for development now requires ``cython >= 0.28.2`` (:issue:`21688`)
+- Testing pandas now requires ``hypothesis>=3.58`` (:issue:22280).  You can find `the Hypothesis docs here <https://hypothesis.readthedocs.io/en/latest/index.html>`_, and a pandas-specific introduction :ref:`in the contributing guide <using-hypothesis>` .
 -
 
 Other
diff --git a/pandas/conftest.py b/pandas/conftest.py
@@ -450,3 +450,37 @@ def mock():
         return importlib.import_module("unittest.mock")
     else:
         return pytest.importorskip("mock")
+
+
+# ----------------------------------------------------------------
+# Global setup for tests using Hypothesis
+
+from hypothesis import strategies as st
+
+# Registering these strategies makes them globally available via st.from_type,
+# which is use for offsets in tests/tseries/offsets/test_offsets_properties.py
+for name in 'MonthBegin MonthEnd BMonthBegin BMonthEnd'.split():
+    cls = getattr(pd.tseries.offsets, name)
+    st.register_type_strategy(cls, st.builds(
+        cls,
+        n=st.integers(-99, 99),
+        normalize=st.booleans(),
+    ))
+
+for name in 'YearBegin YearEnd BYearBegin BYearEnd'.split():
+    cls = getattr(pd.tseries.offsets, name)
+    st.register_type_strategy(cls, st.builds(
+        cls,
+        n=st.integers(-5, 5),
+        normalize=st.booleans(),
+        month=st.integers(min_value=1, max_value=12),
+    ))
+
+for name in 'QuarterBegin QuarterEnd BQuarterBegin BQuarterEnd'.split():
+    cls = getattr(pd.tseries.offsets, name)
+    st.register_type_strategy(cls, st.builds(
+        cls,
+        n=st.integers(-24, 24),
+        normalize=st.booleans(),
+        startingMonth=st.integers(min_value=1, max_value=12)
+    ))
diff --git a/pandas/tests/tseries/offsets/test_offsets_properties.py b/pandas/tests/tseries/offsets/test_offsets_properties.py
@@ -0,0 +1,104 @@
+# -*- coding: utf-8 -*-
+"""
+Behavioral based tests for offsets and date_range.
+
+This file is adapted from https://github.com/pandas-dev/pandas/pull/18761 -
+which was more ambitious but less idiomatic in its use of Hypothesis.
+
+You may wish to consult the previous version for inspiration on further
+tests, or when trying to pin down the bugs exposed by the tests below.
+"""
+
+import pytest
+from hypothesis import given, assume, strategies as st
+from hypothesis.extra.pytz import timezones as pytz_timezones
+from hypothesis.extra.dateutil import timezones as dateutil_timezones
+
+import pandas as pd
+
+from pandas.tseries.offsets import (
+    MonthEnd, MonthBegin, BMonthEnd, BMonthBegin,
+    QuarterEnd, QuarterBegin, BQuarterEnd, BQuarterBegin,
+    YearEnd, YearBegin, BYearEnd, BYearBegin,
+)
+
+# ----------------------------------------------------------------
+# Helpers for generating random data
+
+gen_date_range = st.builds(
+    pd.date_range,
+    start=st.datetimes(
+        # TODO: Choose the min/max values more systematically
+        min_value=pd.Timestamp(1900, 1, 1).to_pydatetime(),
+        max_value=pd.Timestamp(2100, 1, 1).to_pydatetime()
+    ),
+    periods=st.integers(min_value=2, max_value=100),
+    freq=st.sampled_from('Y Q M D H T s ms us ns'.split()),
+    tz=st.one_of(st.none(), dateutil_timezones(), pytz_timezones()),
+)
+
+gen_random_datetime = st.datetimes(
+    min_value=pd.Timestamp.min.to_pydatetime(),
+    max_value=pd.Timestamp.max.to_pydatetime(),
+    timezones=st.one_of(st.none(), dateutil_timezones(), pytz_timezones())
+)
+
+# The strategy for each type is registered in conftest.py, as they don't carry
+# enough runtime information (e.g. type hints) to infer how to build them.
+gen_yqm_offset = st.one_of(*map(st.from_type, [
+    MonthBegin, MonthEnd, BMonthBegin, BMonthEnd,
+    QuarterBegin, QuarterEnd, BQuarterBegin, BQuarterEnd,
+    YearBegin, YearEnd, BYearBegin, BYearEnd
+]))
+
+
+# ----------------------------------------------------------------
+# Offset-specific behaviour tests
+
+
+# Based on CI runs: Always passes on OSX, fails on Linux, sometimes on Windows
+@pytest.mark.xfail(strict=False, reason='inconsistent between OSs, Pythons')
+@given(gen_random_datetime, gen_yqm_offset)
+def test_on_offset_implementations(dt, offset):
+    assume(not offset.normalize)
+    # check that the class-specific implementations of onOffset match
+    # the general case definition:
+    #   (dt + offset) - offset == dt
+    compare = (dt + offset) - offset
+    assert offset.onOffset(dt) == (compare == dt)
+
+
+@pytest.mark.xfail(strict=True)
+@given(gen_yqm_offset, gen_date_range)
+def test_apply_index_implementations(offset, rng):
+    # offset.apply_index(dti)[i] should match dti[i] + offset
+    assume(offset.n != 0)  # TODO: test for that case separately
+
+    # rng = pd.date_range(start='1/1/2000', periods=100000, freq='T')
+    ser = pd.Series(rng)
+
+    res = rng + offset
+    res_v2 = offset.apply_index(rng)
+    assert (res == res_v2).all()
+
+    assert res[0] == rng[0] + offset
+    assert res[-1] == rng[-1] + offset
+    res2 = ser + offset
+    # apply_index is only for indexes, not series, so no res2_v2
+    assert res2.iloc[0] == ser.iloc[0] + offset
+    assert res2.iloc[-1] == ser.iloc[-1] + offset
+    # TODO: Check randomly assorted entries, not just first/last
+
+
+@pytest.mark.xfail(strict=True)
+@given(gen_yqm_offset)
+def test_shift_across_dst(offset):
+    # GH#18319 check that 1) timezone is correctly normalized and
+    # 2) that hour is not incorrectly changed by this normalization
+    # Note that dti includes a transition across DST boundary
+    dti = pd.date_range(start='2017-10-30 12:00:00', end='2017-11-06',
+                        freq='D', tz='US/Eastern')
+    assert (dti.hour == 12).all()  # we haven't screwed up yet
+
+    res = dti + offset
+    assert (res.hour == 12).all()
diff --git a/pandas/tests/tseries/offsets/test_ticks.py b/pandas/tests/tseries/offsets/test_ticks.py
diff --git a/pandas/util/_tester.py b/pandas/util/_tester.py
