diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 4e8a8d654bb2..94c4158e26bb 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -210,3 +210,16 @@ jobs: testResultsFiles: '**/test-*.xml' failTaskOnFailedTests: true testRunTitle: 'Publish test results for Python $(PYTHON_VERSION) $(BITS)-bit $(TEST_MODE) Windows' + +- job: Linux_PyPy3 + pool: + vmIMage: 'ubuntu-16.04' + steps: + - script: source tools/pypy-test.sh + displayName: 'Run PyPy3 Build / Tests' + continueOnError: true + - task: PublishTestResults@2 + inputs: + testResultsFiles: '**/test-*.xml' + testRunTitle: 'Publish test results for PyPy3' + failTaskOnFailedTests: true diff --git a/numpy/core/_internal.py b/numpy/core/_internal.py index 82035e561adc..ab5a64a1aa78 100644 --- a/numpy/core/_internal.py +++ b/numpy/core/_internal.py @@ -8,6 +8,7 @@ import re import sys +import platform from numpy.compat import unicode from .multiarray import dtype, array, ndarray @@ -16,6 +17,8 @@ except ImportError: ctypes = None +IS_PYPY = platform.python_implementation() == 'PyPy' + if (sys.byteorder == 'little'): _nbo = b'<' else: @@ -865,7 +868,12 @@ def npy_ctypes_check(cls): try: # ctypes class are new-style, so have an __mro__. This probably fails # for ctypes classes with multiple inheritance. - ctype_base = cls.__mro__[-2] + if IS_PYPY: + # (..., _ctypes.basics._CData, Bufferable, object) + ctype_base = cls.__mro__[-3] + else: + # # (..., _ctypes._CData, object) + ctype_base = cls.__mro__[-2] # right now, they're part of the _ctypes module return 'ctypes' in ctype_base.__module__ except Exception: diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index c45029599bd4..b29daa675fb3 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -36,7 +36,7 @@ assert_, assert_raises, assert_warns, assert_equal, assert_almost_equal, assert_array_equal, assert_raises_regex, assert_array_almost_equal, assert_allclose, IS_PYPY, HAS_REFCOUNT, assert_array_less, runstring, - temppath, suppress_warnings + temppath, suppress_warnings, break_cycles, ) from numpy.core.tests._locales import CommaDecimalPointLocale @@ -131,6 +131,7 @@ def test_writeable_from_buffer(self): assert_(vals.flags.writeable) @pytest.mark.skipif(sys.version_info[0] < 3, reason="Python 2 always copies") + @pytest.mark.skipif(IS_PYPY, reason="PyPy always copies") def test_writeable_pickle(self): import pickle # Small arrays will be copied without setting base. @@ -3783,7 +3784,7 @@ def test_roundtrip(self): a, pickle.loads(pickle.dumps(a, protocol=proto)), err_msg="%r" % a) del a, DATA, carray - gc.collect() + break_cycles() # check for reference leaks (gh-12793) for ref in refs: assert ref() is None @@ -7180,7 +7181,7 @@ def test_mem_seteventhook(self): # needs to be larger then limit of small memory cacher in ctors.c a = np.zeros(1000) del a - gc.collect() + break_cycles() _multiarray_tests.test_pydatamem_seteventhook_end() class TestMapIter(object): @@ -7752,12 +7753,12 @@ def test_ctypes_data_as_holds_reference(self, arr): # `ctypes_ptr` should hold onto `arr` del arr - gc.collect() + break_cycles() assert_(arr_ref() is not None, "ctypes pointer did not hold onto a reference") # but when the `ctypes_ptr` object dies, so should `arr` del ctypes_ptr - gc.collect() + break_cycles() assert_(arr_ref() is None, "unknowable whether ctypes pointer holds a reference") @@ -7939,15 +7940,15 @@ class Dummy(object): pass assert_(isinstance(obj_subarray, RaisesInFinalize)) # reference should still be held by obj_arr - gc.collect() + break_cycles() assert_(obj_ref() is not None, "object should not already be dead") del obj_arr - gc.collect() + break_cycles() assert_(obj_ref() is not None, "obj_arr should not hold the last reference") del obj_subarray - gc.collect() + break_cycles() assert_(obj_ref() is None, "no references should remain") diff --git a/numpy/core/tests/test_numerictypes.py b/numpy/core/tests/test_numerictypes.py index 71f7b7150614..d0ff5578a0af 100644 --- a/numpy/core/tests/test_numerictypes.py +++ b/numpy/core/tests/test_numerictypes.py @@ -5,7 +5,7 @@ import pytest import numpy as np -from numpy.testing import assert_, assert_equal, assert_raises +from numpy.testing import assert_, assert_equal, assert_raises, IS_PYPY # This is the structure of the table used for plain objects: # @@ -491,6 +491,7 @@ def test_issctype(rep, expected): @pytest.mark.skipif(sys.flags.optimize > 1, reason="no docstrings present to inspect when PYTHONOPTIMIZE/Py_OptimizeFlag > 1") +@pytest.mark.xfail(IS_PYPY, reason="PyPy does not modify tp_doc") class TestDocStrings(object): def test_platform_dependent_aliases(self): if np.int64 is np.int_: diff --git a/numpy/f2py/tests/test_block_docstring.py b/numpy/f2py/tests/test_block_docstring.py index 8fc072a5ed16..4f1678980f8b 100644 --- a/numpy/f2py/tests/test_block_docstring.py +++ b/numpy/f2py/tests/test_block_docstring.py @@ -4,7 +4,7 @@ import pytest from . import util -from numpy.testing import assert_equal +from numpy.testing import assert_equal, IS_PYPY class TestBlockDocString(util.F2PyTest): code = """ @@ -18,6 +18,7 @@ class TestBlockDocString(util.F2PyTest): @pytest.mark.skipif(sys.platform=='win32', reason='Fails with MinGW64 Gfortran (Issue #9673)') + @pytest.mark.xfail(IS_PYPY, reason="PyPy does not modify tp_doc") def test_block_docstring(self): expected = "'i'-array(2,3)\n" assert_equal(self.module.block.__doc__, expected) diff --git a/numpy/lib/tests/test_function_base.py b/numpy/lib/tests/test_function_base.py index 1e04bfaec71c..e2c24a123a36 100644 --- a/numpy/lib/tests/test_function_base.py +++ b/numpy/lib/tests/test_function_base.py @@ -11,7 +11,7 @@ from numpy import ma from numpy.testing import ( assert_, assert_equal, assert_array_equal, assert_almost_equal, - assert_array_almost_equal, assert_raises, assert_allclose, + assert_array_almost_equal, assert_raises, assert_allclose, IS_PYPY, assert_warns, assert_raises_regex, suppress_warnings, HAS_REFCOUNT, ) import numpy.lib.function_base as nfb @@ -3177,6 +3177,7 @@ def test_string_arg(self): class TestAdd_newdoc(object): @pytest.mark.skipif(sys.flags.optimize == 2, reason="Python running -OO") + @pytest.mark.xfail(IS_PYPY, reason="PyPy does not modify tp_doc") def test_add_doc(self): # test np.add_newdoc tgt = "Current flat index into the array." diff --git a/numpy/testing/_private/utils.py b/numpy/testing/_private/utils.py index 1f7b516b3ed6..24e26d65c1b6 100644 --- a/numpy/testing/_private/utils.py +++ b/numpy/testing/_private/utils.py @@ -6,6 +6,7 @@ import os import sys +import platform import re import gc import operator @@ -39,6 +40,7 @@ 'SkipTest', 'KnownFailureException', 'temppath', 'tempdir', 'IS_PYPY', 'HAS_REFCOUNT', 'suppress_warnings', 'assert_array_compare', '_assert_valid_refcount', '_gen_alignment_data', 'assert_no_gc_cycles', + 'break_cycles', ] @@ -50,7 +52,7 @@ class KnownFailureException(Exception): KnownFailureTest = KnownFailureException # backwards compat verbose = 0 -IS_PYPY = '__pypy__' in sys.modules +IS_PYPY = platform.python_implementation() == 'PyPy' HAS_REFCOUNT = getattr(sys, 'getrefcount', None) is not None @@ -2245,6 +2247,7 @@ def _assert_no_gc_cycles_context(name=None): # not meaningful to test if there is no refcounting if not HAS_REFCOUNT: + yield return assert_(gc.isenabled()) @@ -2323,3 +2326,19 @@ def assert_no_gc_cycles(*args, **kwargs): args = args[1:] with _assert_no_gc_cycles_context(name=func.__name__): func(*args, **kwargs) + +def break_cycles(): + """ + Break reference cycles by calling gc.collect + Objects can call other objects' methods (for instance, another object's + __del__) inside their own __del__. On PyPy, the interpreter only runs + between calls to gc.collect, so multiple calls are needed to completely + release all cycles. + """ + + gc.collect() + if IS_PYPY: + # interpreter runs now, to call deleted objects' __del__ methods + gc.collect() + # one more, just to make sure + gc.collect() diff --git a/tools/pypy-test.sh b/tools/pypy-test.sh new file mode 100755 index 000000000000..61a09cd85507 --- /dev/null +++ b/tools/pypy-test.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash + +# Exit if a command fails +set -e +set -o pipefail +# Print expanded commands +set -x + +sudo apt-get -yq update +sudo apt-get -yq install libatlas-base-dev liblapack-dev gfortran-5 +F77=gfortran-5 F90=gfortran-5 \ + +# Download the proper OpenBLAS x64 precompiled library +OPENBLAS=openblas-v0.3.5-manylinux1_x86_64.tar.gz +echo getting $OPENBLAS +wget -q https://3f23b170c54c2533c070-1c8a9b3114517dc5fe17b7c3f8c63a43.ssl.cf2.rackcdn.com/$OPENBLAS -O openblas.tar.gz +mkdir -p openblas +(cd openblas; tar -xf ../openblas.tar.gz) +export LD_LIBRARY_PATH=$PWD/openblas/usr/local/lib +export LIB=$PWD/openblas/usr/local/lib +export INCLUDE=$PWD/openblas/usr/local/include + +# Use a site.cfg to build with local openblas +cat << EOF > site.cfg +[openblas] +libraries = openblas +library_dirs = $PWD/openblas/usr/local/lib:$LIB +include_dirs = $PWD/openblas/usr/local/lib:$LIB +runtime_library_dirs = $PWD/openblas/usr/local/lib +EOF + +echo getting PyPy 3.6 nightly +wget -q http://buildbot.pypy.org/nightly/py3.6/pypy-c-jit-latest-linux64.tar.bz2 -O pypy.tar.bz2 +mkdir -p pypy3 +(cd pypy3; tar --strip-components=1 -xf ../pypy.tar.bz2) +pypy3/bin/pypy3 -mensurepip +pypy3/bin/pypy3 -m pip install --upgrade pip setuptools +pypy3/bin/pypy3 -m pip install --user cython==0.29.0 pytest pytz --no-warn-script-location + +echo +echo pypy3 version +pypy3/bin/pypy3 -c "import sys; print(sys.version)" +echo + +pypy3/bin/pypy3 runtests.py --show-build-log -- -rsx \ + --junitxml=junit/test-results.xml --durations 10 + +echo Make sure the correct openblas has been linked in +# rework after merging PR #12790 or alternative +TEST_GET_CONFIG="import numpy, ctypes, os +dll = ctypes.CDLL(numpy.core._multiarray_umath.__file__) +get_config = dll.openblas_get_config +get_config.restype=ctypes.c_char_p +res = get_config() +print('OpenBLAS get_config returned', str(res)) +assert b'OpenBLAS 0.3.5' in res" + +pypy3/bin/pip install . +(cd pypy3; bin/pypy3 -c "$TEST_GET_CONFIG")