From c09fadcc8220f4ac80f0dcc4d82d22f58ef2d6c1 Mon Sep 17 00:00:00 2001 From: mattip Date: Wed, 19 Dec 2018 09:25:12 +0200 Subject: [PATCH 1/4] DEV: add pypy3 to azure CI --- .gitignore | 2 ++ azure-pipelines.yml | 11 +++++++++++ numpy/testing/_private/utils.py | 1 + tools/azure-pypy-test.sh | 10 ++++++++++ 4 files changed, 24 insertions(+) create mode 100644 tools/azure-pypy-test.sh diff --git a/.gitignore b/.gitignore index a31d6ea44bd1..1b90490ff5e2 100644 --- a/.gitignore +++ b/.gitignore @@ -176,3 +176,5 @@ benchmarks/numpy cythonize.dat numpy/random/mtrand/mtrand.c numpy/random/mtrand/randint_helpers.pxi +# CI run of PyPy +pypy3.5-latest diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 4e8a8d654bb2..3fe8704f0293 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -210,3 +210,14 @@ jobs: testResultsFiles: '**/test-*.xml' failTaskOnFailedTests: true testRunTitle: 'Publish test results for Python $(PYTHON_VERSION) $(BITS)-bit $(TEST_MODE) Windows' + +- job: Linux_PyPy3_latest_nightly + pool: + vmIMage: 'ubuntu-16.04' + steps: + - script: source tools/azure-pypy-test.sh + displayName: 'Run PyPy3.5 Build / Tests' + - task: PublishTestResults@2 + inputs: + testResultsFiles: '**/test-*.xml' + testRunTitle: 'Publish test results for PyPy3.5' diff --git a/numpy/testing/_private/utils.py b/numpy/testing/_private/utils.py index 1f7b516b3ed6..3ace84415800 100644 --- a/numpy/testing/_private/utils.py +++ b/numpy/testing/_private/utils.py @@ -2245,6 +2245,7 @@ def _assert_no_gc_cycles_context(name=None): # not meaningful to test if there is no refcounting if not HAS_REFCOUNT: + yield return assert_(gc.isenabled()) diff --git a/tools/azure-pypy-test.sh b/tools/azure-pypy-test.sh new file mode 100644 index 000000000000..2194b724d9a6 --- /dev/null +++ b/tools/azure-pypy-test.sh @@ -0,0 +1,10 @@ +wget http://buildbot.pypy.org/nightly/py3.5/pypy-c-jit-latest-linux64.tar.bz2 -O pypy.tar.bz2 +mkdir -p pypy3.5-latest +(cd pypy3.5-latest; tar --strip-components=1 -xf ../pypy.tar.bz2) +pypy3.5-latest/bin/pypy3 -mensurepip +pypy3.5-latest/bin/pypy3 -m pip install --upgrade pip setuptools +pypy3.5-latest/bin/pypy3 -m pip install --user cython==0.29.0 pytest pytz +pypy3.5-latest/bin/pypy3 runtests.py -- -rsx --junitxml=junit/test-results.xml --durations 10 +# do not fail the CI run +echo '' + From e67f8c7f78291cec03de944ed121d3e0f2a89c36 Mon Sep 17 00:00:00 2001 From: mattip Date: Sun, 31 Mar 2019 16:47:49 +0300 Subject: [PATCH 2/4] TEST: update for PyPy --- .gitignore | 2 +- azure-pipelines.yml | 8 ++++---- numpy/core/_internal.py | 8 +++++++- numpy/core/tests/test_multiarray.py | 15 ++++++++------- numpy/core/tests/test_numerictypes.py | 3 ++- numpy/f2py/tests/test_block_docstring.py | 3 ++- numpy/lib/tests/test_function_base.py | 3 ++- tools/azure-pypy-test.sh | 10 ---------- tools/pypy-test.sh | 11 +++++++++++ 9 files changed, 37 insertions(+), 26 deletions(-) delete mode 100644 tools/azure-pypy-test.sh create mode 100755 tools/pypy-test.sh diff --git a/.gitignore b/.gitignore index 1b90490ff5e2..e706ad68016e 100644 --- a/.gitignore +++ b/.gitignore @@ -177,4 +177,4 @@ cythonize.dat numpy/random/mtrand/mtrand.c numpy/random/mtrand/randint_helpers.pxi # CI run of PyPy -pypy3.5-latest +pypy3 diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3fe8704f0293..31e548360a83 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -211,13 +211,13 @@ jobs: failTaskOnFailedTests: true testRunTitle: 'Publish test results for Python $(PYTHON_VERSION) $(BITS)-bit $(TEST_MODE) Windows' -- job: Linux_PyPy3_latest_nightly +- job: Linux_PyPy3 pool: vmIMage: 'ubuntu-16.04' steps: - - script: source tools/azure-pypy-test.sh - displayName: 'Run PyPy3.5 Build / Tests' + - script: source tools/pypy-test.sh + displayName: 'Run PyPy3 Build / Tests' - task: PublishTestResults@2 inputs: testResultsFiles: '**/test-*.xml' - testRunTitle: 'Publish test results for PyPy3.5' + testRunTitle: 'Publish test results for PyPy3' diff --git a/numpy/core/_internal.py b/numpy/core/_internal.py index 82035e561adc..1100fbf0e5a5 100644 --- a/numpy/core/_internal.py +++ b/numpy/core/_internal.py @@ -8,6 +8,7 @@ import re import sys +import platform from numpy.compat import unicode from .multiarray import dtype, array, ndarray @@ -16,6 +17,8 @@ except ImportError: ctypes = None +IS_PYPY = platform.python_implementation() == 'PyPy' + if (sys.byteorder == 'little'): _nbo = b'<' else: @@ -865,7 +868,10 @@ def npy_ctypes_check(cls): try: # ctypes class are new-style, so have an __mro__. This probably fails # for ctypes classes with multiple inheritance. - ctype_base = cls.__mro__[-2] + if IS_PYPY: + ctype_base = cls.__mro__[-3] + else: + ctype_base = cls.__mro__[-2] # right now, they're part of the _ctypes module return 'ctypes' in ctype_base.__module__ except Exception: diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index c45029599bd4..5bab992fa608 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -131,6 +131,7 @@ def test_writeable_from_buffer(self): assert_(vals.flags.writeable) @pytest.mark.skipif(sys.version_info[0] < 3, reason="Python 2 always copies") + @pytest.mark.skipif(IS_PYPY, reason="PyPy always copies") def test_writeable_pickle(self): import pickle # Small arrays will be copied without setting base. @@ -3783,7 +3784,7 @@ def test_roundtrip(self): a, pickle.loads(pickle.dumps(a, protocol=proto)), err_msg="%r" % a) del a, DATA, carray - gc.collect() + gc.collect(); gc.collect(); gc.collect() # check for reference leaks (gh-12793) for ref in refs: assert ref() is None @@ -7180,7 +7181,7 @@ def test_mem_seteventhook(self): # needs to be larger then limit of small memory cacher in ctors.c a = np.zeros(1000) del a - gc.collect() + gc.collect(); gc.collect(); gc.collect() _multiarray_tests.test_pydatamem_seteventhook_end() class TestMapIter(object): @@ -7752,12 +7753,12 @@ def test_ctypes_data_as_holds_reference(self, arr): # `ctypes_ptr` should hold onto `arr` del arr - gc.collect() + gc.collect(); gc.collect(); gc.collect() assert_(arr_ref() is not None, "ctypes pointer did not hold onto a reference") # but when the `ctypes_ptr` object dies, so should `arr` del ctypes_ptr - gc.collect() + gc.collect(); gc.collect(); gc.collect() assert_(arr_ref() is None, "unknowable whether ctypes pointer holds a reference") @@ -7939,15 +7940,15 @@ class Dummy(object): pass assert_(isinstance(obj_subarray, RaisesInFinalize)) # reference should still be held by obj_arr - gc.collect() + gc.collect(); gc.collect(); gc.collect() assert_(obj_ref() is not None, "object should not already be dead") del obj_arr - gc.collect() + gc.collect(); gc.collect(); gc.collect() assert_(obj_ref() is not None, "obj_arr should not hold the last reference") del obj_subarray - gc.collect() + gc.collect(); gc.collect(); gc.collect() assert_(obj_ref() is None, "no references should remain") diff --git a/numpy/core/tests/test_numerictypes.py b/numpy/core/tests/test_numerictypes.py index 71f7b7150614..d0ff5578a0af 100644 --- a/numpy/core/tests/test_numerictypes.py +++ b/numpy/core/tests/test_numerictypes.py @@ -5,7 +5,7 @@ import pytest import numpy as np -from numpy.testing import assert_, assert_equal, assert_raises +from numpy.testing import assert_, assert_equal, assert_raises, IS_PYPY # This is the structure of the table used for plain objects: # @@ -491,6 +491,7 @@ def test_issctype(rep, expected): @pytest.mark.skipif(sys.flags.optimize > 1, reason="no docstrings present to inspect when PYTHONOPTIMIZE/Py_OptimizeFlag > 1") +@pytest.mark.xfail(IS_PYPY, reason="PyPy does not modify tp_doc") class TestDocStrings(object): def test_platform_dependent_aliases(self): if np.int64 is np.int_: diff --git a/numpy/f2py/tests/test_block_docstring.py b/numpy/f2py/tests/test_block_docstring.py index 8fc072a5ed16..4f1678980f8b 100644 --- a/numpy/f2py/tests/test_block_docstring.py +++ b/numpy/f2py/tests/test_block_docstring.py @@ -4,7 +4,7 @@ import pytest from . import util -from numpy.testing import assert_equal +from numpy.testing import assert_equal, IS_PYPY class TestBlockDocString(util.F2PyTest): code = """ @@ -18,6 +18,7 @@ class TestBlockDocString(util.F2PyTest): @pytest.mark.skipif(sys.platform=='win32', reason='Fails with MinGW64 Gfortran (Issue #9673)') + @pytest.mark.xfail(IS_PYPY, reason="PyPy does not modify tp_doc") def test_block_docstring(self): expected = "'i'-array(2,3)\n" assert_equal(self.module.block.__doc__, expected) diff --git a/numpy/lib/tests/test_function_base.py b/numpy/lib/tests/test_function_base.py index 1e04bfaec71c..e2c24a123a36 100644 --- a/numpy/lib/tests/test_function_base.py +++ b/numpy/lib/tests/test_function_base.py @@ -11,7 +11,7 @@ from numpy import ma from numpy.testing import ( assert_, assert_equal, assert_array_equal, assert_almost_equal, - assert_array_almost_equal, assert_raises, assert_allclose, + assert_array_almost_equal, assert_raises, assert_allclose, IS_PYPY, assert_warns, assert_raises_regex, suppress_warnings, HAS_REFCOUNT, ) import numpy.lib.function_base as nfb @@ -3177,6 +3177,7 @@ def test_string_arg(self): class TestAdd_newdoc(object): @pytest.mark.skipif(sys.flags.optimize == 2, reason="Python running -OO") + @pytest.mark.xfail(IS_PYPY, reason="PyPy does not modify tp_doc") def test_add_doc(self): # test np.add_newdoc tgt = "Current flat index into the array." diff --git a/tools/azure-pypy-test.sh b/tools/azure-pypy-test.sh deleted file mode 100644 index 2194b724d9a6..000000000000 --- a/tools/azure-pypy-test.sh +++ /dev/null @@ -1,10 +0,0 @@ -wget http://buildbot.pypy.org/nightly/py3.5/pypy-c-jit-latest-linux64.tar.bz2 -O pypy.tar.bz2 -mkdir -p pypy3.5-latest -(cd pypy3.5-latest; tar --strip-components=1 -xf ../pypy.tar.bz2) -pypy3.5-latest/bin/pypy3 -mensurepip -pypy3.5-latest/bin/pypy3 -m pip install --upgrade pip setuptools -pypy3.5-latest/bin/pypy3 -m pip install --user cython==0.29.0 pytest pytz -pypy3.5-latest/bin/pypy3 runtests.py -- -rsx --junitxml=junit/test-results.xml --durations 10 -# do not fail the CI run -echo '' - diff --git a/tools/pypy-test.sh b/tools/pypy-test.sh new file mode 100755 index 000000000000..bcadc1d6deab --- /dev/null +++ b/tools/pypy-test.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +apt-get -yq update +apt-get -yq install libatlas-dev libatlas-base-dev liblapack-dev +wget http://buildbot.pypy.org/nightly/py3.6/pypy-c-jit-latest-linux64.tar.bz2 -O pypy.tar.bz2 +mkdir -p pypy3 +(cd pypy3; tar --strip-components=1 -xf ../pypy.tar.bz2) +pypy3/bin/pypy3 -mensurepip +pypy3/bin/pypy3 -m pip install --upgrade pip setuptools +pypy3/bin/pypy3 -m pip install --user cython==0.29.0 pytest pytz --no-warn-script-location +pypy3/bin/pypy3 runtests.py -- -rsx --junitxml=junit/test-results.xml --durations 10 From a641ef245a9f8d320fac5cdea5632649db5fab4a Mon Sep 17 00:00:00 2001 From: mattip Date: Wed, 17 Apr 2019 11:33:54 +0300 Subject: [PATCH 3/4] MAINT: move gc.collect to function (from review) --- numpy/core/tests/test_multiarray.py | 16 ++++++++-------- numpy/testing/_private/utils.py | 20 +++++++++++++++++++- tools/pypy-test.sh | 4 ++++ 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 5bab992fa608..b29daa675fb3 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -36,7 +36,7 @@ assert_, assert_raises, assert_warns, assert_equal, assert_almost_equal, assert_array_equal, assert_raises_regex, assert_array_almost_equal, assert_allclose, IS_PYPY, HAS_REFCOUNT, assert_array_less, runstring, - temppath, suppress_warnings + temppath, suppress_warnings, break_cycles, ) from numpy.core.tests._locales import CommaDecimalPointLocale @@ -3784,7 +3784,7 @@ def test_roundtrip(self): a, pickle.loads(pickle.dumps(a, protocol=proto)), err_msg="%r" % a) del a, DATA, carray - gc.collect(); gc.collect(); gc.collect() + break_cycles() # check for reference leaks (gh-12793) for ref in refs: assert ref() is None @@ -7181,7 +7181,7 @@ def test_mem_seteventhook(self): # needs to be larger then limit of small memory cacher in ctors.c a = np.zeros(1000) del a - gc.collect(); gc.collect(); gc.collect() + break_cycles() _multiarray_tests.test_pydatamem_seteventhook_end() class TestMapIter(object): @@ -7753,12 +7753,12 @@ def test_ctypes_data_as_holds_reference(self, arr): # `ctypes_ptr` should hold onto `arr` del arr - gc.collect(); gc.collect(); gc.collect() + break_cycles() assert_(arr_ref() is not None, "ctypes pointer did not hold onto a reference") # but when the `ctypes_ptr` object dies, so should `arr` del ctypes_ptr - gc.collect(); gc.collect(); gc.collect() + break_cycles() assert_(arr_ref() is None, "unknowable whether ctypes pointer holds a reference") @@ -7940,15 +7940,15 @@ class Dummy(object): pass assert_(isinstance(obj_subarray, RaisesInFinalize)) # reference should still be held by obj_arr - gc.collect(); gc.collect(); gc.collect() + break_cycles() assert_(obj_ref() is not None, "object should not already be dead") del obj_arr - gc.collect(); gc.collect(); gc.collect() + break_cycles() assert_(obj_ref() is not None, "obj_arr should not hold the last reference") del obj_subarray - gc.collect(); gc.collect(); gc.collect() + break_cycles() assert_(obj_ref() is None, "no references should remain") diff --git a/numpy/testing/_private/utils.py b/numpy/testing/_private/utils.py index 3ace84415800..24e26d65c1b6 100644 --- a/numpy/testing/_private/utils.py +++ b/numpy/testing/_private/utils.py @@ -6,6 +6,7 @@ import os import sys +import platform import re import gc import operator @@ -39,6 +40,7 @@ 'SkipTest', 'KnownFailureException', 'temppath', 'tempdir', 'IS_PYPY', 'HAS_REFCOUNT', 'suppress_warnings', 'assert_array_compare', '_assert_valid_refcount', '_gen_alignment_data', 'assert_no_gc_cycles', + 'break_cycles', ] @@ -50,7 +52,7 @@ class KnownFailureException(Exception): KnownFailureTest = KnownFailureException # backwards compat verbose = 0 -IS_PYPY = '__pypy__' in sys.modules +IS_PYPY = platform.python_implementation() == 'PyPy' HAS_REFCOUNT = getattr(sys, 'getrefcount', None) is not None @@ -2324,3 +2326,19 @@ def assert_no_gc_cycles(*args, **kwargs): args = args[1:] with _assert_no_gc_cycles_context(name=func.__name__): func(*args, **kwargs) + +def break_cycles(): + """ + Break reference cycles by calling gc.collect + Objects can call other objects' methods (for instance, another object's + __del__) inside their own __del__. On PyPy, the interpreter only runs + between calls to gc.collect, so multiple calls are needed to completely + release all cycles. + """ + + gc.collect() + if IS_PYPY: + # interpreter runs now, to call deleted objects' __del__ methods + gc.collect() + # one more, just to make sure + gc.collect() diff --git a/tools/pypy-test.sh b/tools/pypy-test.sh index bcadc1d6deab..3dd3b439f235 100755 --- a/tools/pypy-test.sh +++ b/tools/pypy-test.sh @@ -8,4 +8,8 @@ mkdir -p pypy3 pypy3/bin/pypy3 -mensurepip pypy3/bin/pypy3 -m pip install --upgrade pip setuptools pypy3/bin/pypy3 -m pip install --user cython==0.29.0 pytest pytz --no-warn-script-location +echo +echo pypy3 version +pypy3/bin/pypy3 -c "import sys; print(sys.version)" +echo pypy3/bin/pypy3 runtests.py -- -rsx --junitxml=junit/test-results.xml --durations 10 From f402a691a9c99e0ba2e090d4f8269ea81aac1a50 Mon Sep 17 00:00:00 2001 From: mattip Date: Sat, 20 Apr 2019 22:39:24 +0300 Subject: [PATCH 4/4] MAINT: use openblas, gfortran, tweak azure and comment mro (from review) --- .gitignore | 2 -- azure-pipelines.yml | 2 ++ numpy/core/_internal.py | 2 ++ tools/pypy-test.sh | 52 +++++++++++++++++++++++++++++++++++++---- 4 files changed, 52 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index e706ad68016e..a31d6ea44bd1 100644 --- a/.gitignore +++ b/.gitignore @@ -176,5 +176,3 @@ benchmarks/numpy cythonize.dat numpy/random/mtrand/mtrand.c numpy/random/mtrand/randint_helpers.pxi -# CI run of PyPy -pypy3 diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 31e548360a83..94c4158e26bb 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -217,7 +217,9 @@ jobs: steps: - script: source tools/pypy-test.sh displayName: 'Run PyPy3 Build / Tests' + continueOnError: true - task: PublishTestResults@2 inputs: testResultsFiles: '**/test-*.xml' testRunTitle: 'Publish test results for PyPy3' + failTaskOnFailedTests: true diff --git a/numpy/core/_internal.py b/numpy/core/_internal.py index 1100fbf0e5a5..ab5a64a1aa78 100644 --- a/numpy/core/_internal.py +++ b/numpy/core/_internal.py @@ -869,8 +869,10 @@ def npy_ctypes_check(cls): # ctypes class are new-style, so have an __mro__. This probably fails # for ctypes classes with multiple inheritance. if IS_PYPY: + # (..., _ctypes.basics._CData, Bufferable, object) ctype_base = cls.__mro__[-3] else: + # # (..., _ctypes._CData, object) ctype_base = cls.__mro__[-2] # right now, they're part of the _ctypes module return 'ctypes' in ctype_base.__module__ diff --git a/tools/pypy-test.sh b/tools/pypy-test.sh index 3dd3b439f235..61a09cd85507 100755 --- a/tools/pypy-test.sh +++ b/tools/pypy-test.sh @@ -1,15 +1,59 @@ #!/usr/bin/env bash -apt-get -yq update -apt-get -yq install libatlas-dev libatlas-base-dev liblapack-dev -wget http://buildbot.pypy.org/nightly/py3.6/pypy-c-jit-latest-linux64.tar.bz2 -O pypy.tar.bz2 +# Exit if a command fails +set -e +set -o pipefail +# Print expanded commands +set -x + +sudo apt-get -yq update +sudo apt-get -yq install libatlas-base-dev liblapack-dev gfortran-5 +F77=gfortran-5 F90=gfortran-5 \ + +# Download the proper OpenBLAS x64 precompiled library +OPENBLAS=openblas-v0.3.5-manylinux1_x86_64.tar.gz +echo getting $OPENBLAS +wget -q https://3f23b170c54c2533c070-1c8a9b3114517dc5fe17b7c3f8c63a43.ssl.cf2.rackcdn.com/$OPENBLAS -O openblas.tar.gz +mkdir -p openblas +(cd openblas; tar -xf ../openblas.tar.gz) +export LD_LIBRARY_PATH=$PWD/openblas/usr/local/lib +export LIB=$PWD/openblas/usr/local/lib +export INCLUDE=$PWD/openblas/usr/local/include + +# Use a site.cfg to build with local openblas +cat << EOF > site.cfg +[openblas] +libraries = openblas +library_dirs = $PWD/openblas/usr/local/lib:$LIB +include_dirs = $PWD/openblas/usr/local/lib:$LIB +runtime_library_dirs = $PWD/openblas/usr/local/lib +EOF + +echo getting PyPy 3.6 nightly +wget -q http://buildbot.pypy.org/nightly/py3.6/pypy-c-jit-latest-linux64.tar.bz2 -O pypy.tar.bz2 mkdir -p pypy3 (cd pypy3; tar --strip-components=1 -xf ../pypy.tar.bz2) pypy3/bin/pypy3 -mensurepip pypy3/bin/pypy3 -m pip install --upgrade pip setuptools pypy3/bin/pypy3 -m pip install --user cython==0.29.0 pytest pytz --no-warn-script-location + echo echo pypy3 version pypy3/bin/pypy3 -c "import sys; print(sys.version)" echo -pypy3/bin/pypy3 runtests.py -- -rsx --junitxml=junit/test-results.xml --durations 10 + +pypy3/bin/pypy3 runtests.py --show-build-log -- -rsx \ + --junitxml=junit/test-results.xml --durations 10 + +echo Make sure the correct openblas has been linked in +# rework after merging PR #12790 or alternative +TEST_GET_CONFIG="import numpy, ctypes, os +dll = ctypes.CDLL(numpy.core._multiarray_umath.__file__) +get_config = dll.openblas_get_config +get_config.restype=ctypes.c_char_p +res = get_config() +print('OpenBLAS get_config returned', str(res)) +assert b'OpenBLAS 0.3.5' in res" + +pypy3/bin/pip install . +(cd pypy3; bin/pypy3 -c "$TEST_GET_CONFIG")