Merge pull request scipy#10254 from tylerjereddy/backport_122_June_3

MAINT: Backports / updates for 1.2.2 (LTS)

.circleci/config.yml

@@ -91,55 +91,6 @@ jobs:
             git commit -m "Docs build of $CIRCLE_SHA1";
             git push --set-upstream origin gh-pages --force
 
-
-  # Run test suite on pypy3
-  pypy3:
-    docker:
-      - image: pypy:3-6.0.0
-
-    steps:
-      - restore_cache:
-          keys:
-            - pypy3-ccache-{{ .Branch }}
-            - pypy3-ccache
-      - checkout
-      - run:
-          name: setup
-          command: |
-            apt-get -yq update
-            apt-get -yq install libatlas-dev libatlas-base-dev liblapack-dev gfortran ccache
-            ccache -M 512M
-            export CCACHE_COMPRESS=1
-            export NPY_NUM_BUILD_JOBS=`pypy3 -c 'import multiprocessing as mp; print(mp.cpu_count())'`
-            export PATH=/usr/lib/ccache:$PATH
-            # XXX: use "numpy>=1.15.0" when it's released
-            pypy3 -mpip install --upgrade pip setuptools wheel
-            pypy3 -mpip install --no-build-isolation --extra-index https://antocuni.github.io/pypy-wheels/ubuntu pytest pytest-xdist Tempita "Cython>=0.28.2" mpmath
-            pypy3 -mpip install --no-build-isolation git+https://github.com/numpy/numpy.git@db552b5b6b37f2ff085b304751d7a2ebed26adc9
-      - run:
-          name: build
-          command: |
-            export CCACHE_COMPRESS=1
-            export PATH=/usr/lib/ccache:$PATH
-            # Limit parallelism for Cythonization to 4 processes, to
-            # avoid exceeding CircleCI memory limits
-            export SCIPY_NUM_CYTHONIZE_JOBS=4
-            export NPY_NUM_BUILD_JOBS=`pypy3 -c 'import multiprocessing as mp; print(mp.cpu_count())'`
-            # Less aggressive optimization flags for faster compilation
-            OPT="-O1" FOPT="-O1" pypy3 setup.py build
-      - save_cache:
-          key: pypy3-ccache-{{ .Branch }}-{{ .BuildNum }}
-          paths:
-            - ~/.ccache
-            - ~/.cache/pip
-      - run:
-          name: test
-          command: |
-            # CircleCI has 4G memory limit, play it safe
-            export SCIPY_AVAILABLE_MEM=1G
-            pypy3 runtests.py -- -rfEX -n 3 --durations=30
-
-
 workflows:
   version: 2
   default:
@@ -152,4 +103,3 @@ workflows:
           filters:
             branches:
               only: master
-      - pypy3

.travis.yml

@@ -117,13 +117,13 @@ before_install:
       export CFLAGS="-arch x86_64"
       export CXXFLAGS="-arch x86_64"
       printenv
-      wget https://3f23b170c54c2533c070-1c8a9b3114517dc5fe17b7c3f8c63a43.ssl.cf2.rackcdn.com/openblas-0.2.20-345-g6a6ffaff-Darwin-x86_64-1becaaa.tar.gz -O openblas.tar.gz
+      wget https://3f23b170c54c2533c070-1c8a9b3114517dc5fe17b7c3f8c63a43.ssl.cf2.rackcdn.com/openblas-v0.3.5-274-g6a8b4269-macosx_10_9_x86_64-gf_1becaaa.tar.gz -O openblas.tar.gz
       mkdir openblas
       tar -xzf openblas.tar.gz -C openblas
       # Modify the openblas dylib so it can be used in its current location
       # Also make it use the current install location for libgfortran, libquadmath, and libgcc_s.
       pushd openblas/usr/local/lib
-      install_name_tool -id $TRAVIS_BUILD_DIR/openblas/usr/local/lib/libopenblasp-r0.3.0.dev.dylib libopenblas.dylib
+      install_name_tool -id $TRAVIS_BUILD_DIR/openblas/usr/local/lib/libopenblasp-r0.3.7.dev.dylib libopenblas.dylib
       install_name_tool -change /usr/local/gfortran/lib/libgfortran.3.dylib `$FC -v 2>&1 | perl -nle 'print $1 if m{--libdir=([^\s]+)}'`/libgfortran.3.dylib libopenblas.dylib
       install_name_tool -change /usr/local/gfortran/lib/libquadmath.0.dylib `$FC -v 2>&1 | perl -nle 'print $1 if m{--libdir=([^\s]+)}'`/libquadmath.0.dylib libopenblas.dylib
       install_name_tool -change /usr/local/gfortran/lib/libgcc_s.1.dylib `$FC -v 2>&1 | perl -nle 'print $1 if m{--libdir=([^\s]+)}'`/libgcc_s.1.dylib libopenblas.dylib
@@ -147,22 +147,9 @@ before_install:
   - if [ -n "$NUMPYSPEC" ]; then travis_retry pip install $NUMPYSPEC; fi
   - travis_retry pip install --upgrade pytest pytest-xdist pytest-faulthandler mpmath argparse Pillow codecov
   - travis_retry pip install gmpy2  # speeds up mpmath (scipy.special tests)
-  - |
-    if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
-      # optional sparse.linalg dependency (test linux only, no suitesparse installed on osx)
-      if [ -z "$NUMPYSPEC" ]; then
-          # numpy must be installed to build scikit-umfpack
-          travis_retry pip install numpy
-      fi
-      travis_retry pip install scikit-umfpack
-      if [ -z "$NUMPYSPEC" ]; then
-          # cleanup after ourselves
-          travis_retry pip uninstall -y numpy
-      fi
-    fi
   - |
     if [ "${TESTMODE}" == "full" ]; then
-        travis_retry pip install pytest-cov coverage matplotlib
+        travis_retry pip install pytest-cov coverage matplotlib scikit-umfpack
     fi
   - |
     if [ "${REFGUIDE_CHECK}" == "1" ]; then

appveyor.yml

@@ -12,8 +12,8 @@ environment:
   global:
       MINGW_32: C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin
       MINGW_64: C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin
-      OPENBLAS_32: https://3f23b170c54c2533c070-1c8a9b3114517dc5fe17b7c3f8c63a43.ssl.cf2.rackcdn.com/openblas-5f998ef_gcc7_1_0_win32.zip
-      OPENBLAS_64: https://3f23b170c54c2533c070-1c8a9b3114517dc5fe17b7c3f8c63a43.ssl.cf2.rackcdn.com/openblas-5f998ef_gcc7_1_0_win64.zip
+      OPENBLAS_32: https://3f23b170c54c2533c070-1c8a9b3114517dc5fe17b7c3f8c63a43.ssl.cf2.rackcdn.com/openblas-v0.3.5-274-g6a8b4269-win32-gcc_7_1_0.zip
+      OPENBLAS_64: https://3f23b170c54c2533c070-1c8a9b3114517dc5fe17b7c3f8c63a43.ssl.cf2.rackcdn.com/openblas-v0.3.5-274-g6a8b4269-win_amd64-gcc_7_1_0.zip
       NUMPY_HEAD: https://github.com/numpy/numpy.git
       NUMPY_BRANCH: master
       APPVEYOR_SAVE_CACHE_ON_ERROR: true
@@ -137,17 +137,6 @@ install:
   # Install the scipy test dependencies.
   - '%CMD_IN_ENV% pip install -U --timeout 5 --retries 2 -r tools/ci/appveyor/requirements.txt'
 
-  # Replace numpy distutils with a version that can build with msvc + mingw-gfortran
-  - ps: |
-      $NumpyDir = $((python -c 'import os; import numpy; print(os.path.dirname(numpy.__file__))') | Out-String).Trim()
-      rm -r -Force "$NumpyDir\distutils"
-      $tmpdir = New-TemporaryFile | %{ rm $_; mkdir $_ }
-
-      echo $env:NUMPY_HEAD
-      echo $env:NUMPY_BRANCH
-      git clone -q --depth=1 -b $env:NUMPY_BRANCH $env:NUMPY_HEAD $tmpdir
-      mv $tmpdir\numpy\distutils $NumpyDir
-
 build_script:
   - ps: |
       $PYTHON_ARCH = $env:PYTHON_ARCH

azure-pipelines.yml

@@ -16,12 +16,12 @@ jobs:
            docker run -v $(pwd):/scipy i386/ubuntu:bionic /bin/bash -c "cd scipy && \
            apt-get -y update && \
            apt-get -y install python3.6-dev python3-pip pkg-config libpng-dev libjpeg8-dev libfreetype6-dev && \
-           pip3 install setuptools wheel numpy cython==0.29 pytest pytest-timeout pytest-xdist pytest-env pytest-faulthandler Pillow mpmath matplotlib && \
+           pip3 install setuptools wheel 'numpy<1.17' cython==0.29 pytest pytest-timeout pytest-xdist pytest-env pytest-faulthandler Pillow mpmath matplotlib && \
            apt-get -y install gfortran-5 wget && \
            cd .. && \
            mkdir openblas && cd openblas && \
-           wget https://3f23b170c54c2533c070-1c8a9b3114517dc5fe17b7c3f8c63a43.ssl.cf2.rackcdn.com/openblas-0.3.0-Linux-i686.tar.gz && \
-           tar zxvf openblas-0.3.0-Linux-i686.tar.gz && \
+           wget https://3f23b170c54c2533c070-1c8a9b3114517dc5fe17b7c3f8c63a43.ssl.cf2.rackcdn.com/openblas-v0.3.5-274-g6a8b4269-manylinux1_i686.tar.gz && \
+           tar zxvf openblas-v0.3.5-274-g6a8b4269-manylinux1_i686.tar.gz && \
            cp -r ./usr/local/lib/* /usr/lib && \
            cp ./usr/local/include/* /usr/include && \
            cd ../scipy && \
@@ -35,8 +35,8 @@ jobs:
   pool:
     vmImage: 'VS2017-Win2016'
   variables:
-      OPENBLAS_32: https://3f23b170c54c2533c070-1c8a9b3114517dc5fe17b7c3f8c63a43.ssl.cf2.rackcdn.com/openblas-5f998ef_gcc7_1_0_win32.zip
-      OPENBLAS_64: https://3f23b170c54c2533c070-1c8a9b3114517dc5fe17b7c3f8c63a43.ssl.cf2.rackcdn.com/openblas-5f998ef_gcc7_1_0_win64.zip
+      OPENBLAS_32: https://3f23b170c54c2533c070-1c8a9b3114517dc5fe17b7c3f8c63a43.ssl.cf2.rackcdn.com/openblas-v0.3.5-274-g6a8b4269-win32-gcc_7_1_0.zip
+      OPENBLAS_64: https://3f23b170c54c2533c070-1c8a9b3114517dc5fe17b7c3f8c63a43.ssl.cf2.rackcdn.com/openblas-v0.3.5-274-g6a8b4269-win_amd64-gcc_7_1_0.zip
   strategy:
     maxParallel: 4
     matrix:
@@ -95,25 +95,18 @@ jobs:
       Write-Host "Python Version: $pyversion"
       $target = "C:\\hostedtoolcache\\windows\\Python\\$pyversion\\$(PYTHON_ARCH)\\lib\\openblas.a"
       Write-Host "target path: $target"
-      cp $tmpdir\$(BITS)\lib\libopenblas_5f998ef_gcc7_1_0.a $target
+      cp $tmpdir\$(BITS)\lib\libopenblas_v0.3.5-274-g6a8b4269-gcc_7_1_0.a $target
     displayName: 'Download / Install OpenBLAS'
   - powershell: |
-      # NOTE: can probably (eventually) abstract this 
-      # upstream in Microsoft repo to support x86 natively
-      choco install -y mingw --forcex86 --force --version=5.3.0
+      choco install -y mingw --forcex86 --force --version=6.4.0
     displayName: 'Install 32-bit mingw for 32-bit builds'
     condition: eq(variables['BITS'], 32)
-  - script: python -m pip install numpy cython==0.28.5 pytest pytest-timeout pytest-xdist pytest-env pytest-faulthandler Pillow mpmath matplotlib
-    displayName: 'Install dependencies'
   - powershell: |
-      # need a version of NumPy distutils that can build
-      # with msvc + mingw-gfortran
-      $NumpyDir = $((python -c 'import os; import numpy; print(os.path.dirname(numpy.__file__))') | Out-String).Trim()
-      rm -r -Force "$NumpyDir\distutils"
-      $tmpdir = New-TemporaryFile | %{ rm $_; mkdir $_ }
-      git clone -q --depth=1 -b master https://github.com/numpy/numpy.git $tmpdir
-      mv $tmpdir\numpy\distutils $NumpyDir
-    displayName: 'Replace NumPy distutils'
+      choco install -y mingw --force --version=6.4.0
+    displayName: 'Install 64-bit mingw for 64-bit builds'
+    condition: eq(variables['BITS'], 64)
+  - bash: python -m pip install 'numpy<1.17' cython==0.28.5 pytest pytest-timeout pytest-xdist pytest-env pytest-faulthandler Pillow mpmath matplotlib
+    displayName: 'Install dependencies'
   - powershell: |
       If ($(BITS) -eq 32) {
           # 32-bit build requires careful adjustments
@@ -122,17 +115,20 @@ jobs:
           $env:NPY_DISTUTILS_APPEND_FLAGS = 1
           $env:CFLAGS = "-m32"
           $env:LDFLAGS = "-m32"
-          $env:PATH = "C:\\tools\\mingw32\\bin;" + $env:PATH
           refreshenv
       }
 
+      $env:PATH = "C:\\ProgramData\\chocolatey\\lib\\mingw\\tools\\install\\mingw$(BITS)\\bin;" + $env:PATH
+
       mkdir dist
       pip wheel --no-build-isolation -v -v -v --wheel-dir=dist .
       ls dist -r | Foreach-Object {
           pip install $_.FullName
       }
     displayName: 'Build SciPy'
-  - script: python runtests.py -n --mode=$(TEST_MODE) -- -n auto -rsx --junitxml=junit/test-results.xml
+  - powershell: |
+      $env:PATH = "C:\\ProgramData\\chocolatey\\lib\\mingw\\tools\\install\\mingw$(BITS)\\bin;" + $env:PATH
+      python runtests.py -n --mode=$(TEST_MODE) -- -n auto -rsx --junitxml=junit/test-results.xml
     displayName: 'Run SciPy Test Suite'
   - task: PublishTestResults@2
     inputs:

doc/release/1.2.2-notes.rst

@@ -5,13 +5,35 @@ SciPy 1.2.2 Release Notes
 .. contents::
 
 SciPy 1.2.2 is a bug-fix release with no new features compared to 1.2.1.
+Importantly, the SciPy 1.2.2 wheels are built with OpenBLAS 0.3.7.dev to
+alleviate issues with SkylakeX AVX512 kernels.
 
 Authors
 =======
 
+* CJ Carey
+* Tyler Dawson +
+* Ralf Gommers
+* Kai Striega
+* Andrew Nelson
+* Tyler Reddy
+* Kevin Sheppard +
+
+A total of 7 people contributed to this release.
+People with a "+" by their names contributed a patch for the first time.
+This list of names is automatically generated, and may not be fully complete.
+
 Issues closed for 1.2.2
 -----------------------
+* `#9611 <https://github.com/scipy/scipy/issues/9611>`__: Overflow error with new way of p-value calculation in kendall tau correlation for perfectly monotonic vectors
+* `#9964 <https://github.com/scipy/scipy/issues/9964>`__: optimize.newton : overwrites x0 argument when it is a numpy array
+* `#9784 <https://github.com/scipy/scipy/issues/9784>`__: TST: Minimum NumPy version is not being CI tested
+* `#10132 <https://github.com/scipy/scipy/issues/10132>`__: Docs: Description of nnz attribute of sparse.csc_matrix misleading
 
 Pull requests for 1.2.2
 -----------------------
-
+* `#10056 <https://github.com/scipy/scipy/pull/10056>`__: BUG: Ensure factorial is not too large in kendaltau
+* `#9991 <https://github.com/scipy/scipy/pull/9991>`__: BUG: Avoid inplace modification of input array in newton
+* `#9788 <https://github.com/scipy/scipy/pull/9788>`__: TST, BUG: f2py-related issues with NumPy < 1.14.0
+* `#9749 <https://github.com/scipy/scipy/pull/9749>`__: BUG: MapWrapper.__exit__ should terminate
+* `#10141 <https://github.com/scipy/scipy/pull/10141>`__: Update description for nnz on csc.py 

scipy/_lib/_util.py

@@ -387,6 +387,7 @@ def __enter__(self):
 
     def __del__(self):
         self.close()
+        self.terminate()
 
     def terminate(self):
         if self._own_pool:
@@ -402,11 +403,8 @@ def close(self):
 
     def __exit__(self, exc_type, exc_value, traceback):
         if self._own_pool:
-            if exc_type is None:
-                self.pool.close()
-                self.pool.join()
-            else:
-                self.pool.terminate()
+            self.pool.close()
+            self.pool.terminate()
 
     def __call__(self, func, iterable):
         # only accept one iterable because that's all Pool.map accepts

scipy/optimize/_differentialevolution.py

@@ -802,10 +802,12 @@ def __enter__(self):
     def __exit__(self, *args):
         # to make sure resources are closed down
         self._mapwrapper.close()
+        self._mapwrapper.terminate()
 
     def __del__(self):
         # to make sure resources are closed down
         self._mapwrapper.close()
+        self._mapwrapper.terminate()
 
     def __next__(self):
         """

scipy/optimize/tests/test__differential_evolution.py

@@ -1,6 +1,8 @@
 """
 Unit tests for the differential global minimization algorithm.
 """
+import multiprocessing
+
 from scipy.optimize import _differentialevolution
 from scipy.optimize._differentialevolution import DifferentialEvolutionSolver
 from scipy.optimize import differential_evolution
@@ -527,8 +529,18 @@ def test_immediate_updating(self):
     def test_parallel(self):
         # smoke test for parallelisation with deferred updating
         bounds = [(0., 2.), (0., 2.)]
-        with DifferentialEvolutionSolver(rosen, bounds,
-                                         updating='deferred',
+        try:
+            p = multiprocessing.Pool(2)
+            with DifferentialEvolutionSolver(rosen, bounds,
+                                             updating='deferred',
+                                             workers=p.map) as solver:
+                assert_(solver._mapwrapper.pool is not None)
+                assert_(solver._updating == 'deferred')
+                solver.solve()
+        finally:
+            p.close()
+
+        with DifferentialEvolutionSolver(rosen, bounds, updating='deferred',
                                          workers=2) as solver:
             assert_(solver._mapwrapper.pool is not None)
             assert_(solver._updating == 'deferred')

scipy/optimize/tests/test_zeros.py

@@ -5,7 +5,8 @@
 
 from numpy.testing import (assert_warns, assert_,
                            assert_allclose,
-                           assert_equal)
+                           assert_equal,
+                           assert_array_equal)
 import numpy as np
 from numpy import finfo, power, nan, isclose
 
@@ -406,6 +407,13 @@ def test_deriv_zero_warning(self):
         dfunc = lambda x: 2*x
         assert_warns(RuntimeWarning, zeros.newton, func, 0.0, dfunc)
 
+    def test_newton_does_not_modify_x0(self):
+        # https://github.com/scipy/scipy/issues/9964
+        x0 = np.array([0.1, 3])
+        x0_copy = x0.copy()  # Copy to test for equality.
+        newton(np.sin, x0, np.cos)
+        assert_array_equal(x0, x0_copy)
+
 
 def test_gh_5555():
     root = 0.1

scipy/optimize/zeros.py

@@ -352,11 +352,13 @@ def _array_newton(func, x0, fprime, args, tol, maxiter, fprime2, full_output):
     Do not use this method directly. This method is called from `newton`
     when ``np.size(x0) > 1`` is ``True``. For docstring, see `newton`.
     """
+    # Explicitly copy `x0` as `p` will be modified inplace, but, the
+    # user's array should not be altered.
     try:
-        p = np.asarray(x0, dtype=float)
+        p = np.array(x0, copy=True, dtype=float)
     except TypeError:
         # can't convert complex to float
-        p = np.asarray(x0)
+        p = np.array(x0, copy=True)
 
     failures = np.ones_like(p, dtype=bool)
     nz_der = np.ones_like(failures)

scipy/stats/stats.py

@@ -3629,11 +3629,10 @@ def count_rank_tie(ranks):
         elif size == 2:
             pvalue = 1.0
         elif c == 0:
-            pvalue = 2.0/np.math.factorial(size)
+            pvalue = 2.0/math.factorial(size) if size < 171 else 0.0
         elif c == 1:
-            pvalue = 2.0/np.math.factorial(size-1)
+            pvalue = 2.0/math.factorial(size-1) if (size-1) < 171 else 0.0
         else:
-            old = [0.0]*(c+1)
             new = [0.0]*(c+1)
             new[0] = 1.0
             new[1] = 1.0
@@ -3643,7 +3642,7 @@ def count_rank_tie(ranks):
                     new[k] += new[k-1]
                 for k in range(j,c+1):
                     new[k] += new[k-1] - old[k-j]
-            pvalue = 2.0*sum(new)/np.math.factorial(size)
+            pvalue = 2.0*sum(new)/math.factorial(size) if size < 171 else 0.0
 
     elif method == 'asymptotic':
         # con_minus_dis is approx normally distributed with this variance [3]_

scipy/stats/tests/test_stats.py

@@ -977,6 +977,20 @@ def test_weightedtau():
     assert_approx_equal(tau, -0.56694968153682723)
 
 
+def test_kendall_tau_large():
+    n = 172.
+    x = np.arange(n)
+    y = np.arange(n)
+    _, pval = stats.kendalltau(x, y, method='exact')
+    assert_equal(pval, 0.0)
+    y[-1], y[-2] = y[-2], y[-1]
+    _, pval = stats.kendalltau(x, y, method='exact')
+    assert_equal(pval, 0.0)
+    y[-3], y[-4] = y[-4], y[-3]
+    _, pval = stats.kendalltau(x, y, method='exact')
+    assert_equal(pval, 0.0)
+
+
 def test_weightedtau_vs_quadratic():
     # Trivial quadratic implementation, all parameters mandatory
     def wkq(x, y, rank, weigher, add):