Update feature/groupby_optimization from master (#843)

* Improve boxing (#832)

* Specify sdc version from channel for examples testing (#837)

* Specify sdc version from channel for examples testing

It occurs that conda resolver can take Intel SDC package
not from first channel where it is found.
Specify particular SDC version to avoid this in examples
for now.
Also print info for environment creation and package installing

* Fix incerrectly used f-string

* Fix log_info call

* Numba 0.49.0 all (#824)

* Fix run tests

Remove import of _getitem_array1d

* expectedFailure

* expectedFailure-2

* expectedFailure-3

* Conda recipe numba==0.49

* expectedFailure-4

* Refactor imports from Numba

* Unskip tests

* Fix using of numpy_support.from_dtype()

* Unskip tests

* Fix DataFrame tests with rewrite IR without Del statements

* Unskip tests

* Fix corr_overload with type inference error for none < 1

* Fix hpat_pandas_series_cov with type inference error for none < 2

* Unskip tests

* Unskip tests

* Fixed iternext_series_array with using _getitem_array1d.

_getitem_array1d is replaced with _getitem_array_single_int in Numba 0.49.

* Unskip tests

* Unskip old test

* Fix Series.at

* Unskip tests

* Add decrefs in boxing (#836)

* Adding extension type for pd.RangeIndex (#820)

* Adding extension type for pd.RangeIndex

This commit adds Numba extension types for pandas.RangeIndex class,
allowing creation of pd.RangeIndex objects and passing and returning them
to/from nopython functions.

* Applying review comments

* Fix for PR 831 (#839)

* Update pyarrow version to 0.17.0

Update recipe, code and docs.

* Disable intel channel

* Disable intel channel for testing

* Fix remarks

Co-authored-by: Vyacheslav Smirnov <vyacheslav.s.smirnov@intel.com>

* Update to Numba 0.49.1 (#838)

* Update to Numba 0.49.1

* Fix requirements.txt

Co-authored-by: Elena Totmenina <totmeninal@mail.ru>
Co-authored-by: Vyacheslav-Smirnov <51660067+Vyacheslav-Smirnov@users.noreply.github.com>
Co-authored-by: Sergey Pokhodenko <sergey.pokhodenko@intel.com>
Co-authored-by: Alexey Kozlov <52973316+kozlov-alexey@users.noreply.github.com>
Co-authored-by: Vyacheslav Smirnov <vyacheslav.s.smirnov@intel.com>

Author: 5 people

README.rst

@@ -23,9 +23,6 @@ the code by leveraging modern hardware instructions and by utilizing all availab
 
 Intel® SDC documentation can be found `here <https://intelpython.github.io/sdc-doc/>`_.
 
-Intel® SDC uses special Numba build based on ``0.48.0`` tag for build and run.
-Required Numba version can be installed from ``intel/label/beta`` channel from the Anaconda Cloud.
-
 .. note::
     For maximum performance and stability, please use numba from ``intel/label/beta`` channel.
 
@@ -61,9 +58,6 @@ If you do not have conda, we recommend using Miniconda3::
     ./miniconda.sh -b
     export PATH=$HOME/miniconda3/bin:$PATH
 
-Intel® SDC uses special Numba build based on ``0.48.0`` tag for build and run.
-Required Numba version can be installed from ``intel/label/beta`` channel from the Anaconda Cloud.
-
 .. note::
     For maximum performance and stability, please use numba from ``intel/label/beta`` channel.
 
@@ -88,7 +82,7 @@ Building on Linux with setuptools
 
     PYVER=<3.6 or 3.7>
     NUMPYVER=<1.16 or 1.17>
-    conda create -n sdc-env -q -y -c intel/label/beta -c defaults -c intel -c conda-forge python=$PYVER numpy=$NUMPYVER numba=0.48.0 pandas=0.25.3 pyarrow=0.15.1 gcc_linux-64 gxx_linux-64
+    conda create -n sdc-env -q -y -c intel/label/beta -c defaults -c intel -c conda-forge python=$PYVER numpy=$NUMPYVER numba=0.49 pandas=0.25.3 pyarrow=0.17.0 gcc_linux-64 gxx_linux-64
     source activate sdc-env
     git clone https://github.com/IntelPython/sdc.git
     cd sdc
@@ -126,7 +120,7 @@ Building on Windows with setuptools
 
     set PYVER=<3.6 or 3.7>
     set NUMPYVER=<1.16 or 1.17>
-    conda create -n sdc-env -c intel/label/beta -c defaults -c intel -c conda-forge python=%PYVER% numpy=%NUMPYVER% numba=0.48.0 pandas=0.25.3 pyarrow=0.15.1
+    conda create -n sdc-env -c intel/label/beta -c defaults -c intel -c conda-forge python=%PYVER% numpy=%NUMPYVER% numba=0.49 pandas=0.25.3 pyarrow=0.17.0
     conda activate sdc-env
     set INCLUDE=%INCLUDE%;%CONDA_PREFIX%\Library\include
     set LIB=%LIB%;%CONDA_PREFIX%\Library\lib

buildscripts/build.py

@@ -35,9 +35,6 @@
 
 def build(sdc_utils):
     os.chdir(str(sdc_utils.src_path))
-    # For Windows build do not use intel channel due to build issue
-    if platform.system() == 'Windows':
-        sdc_utils.channels = '-c intel/label/beta -c defaults -c conda-forge'
 
     sdc_utils.log_info('Start Intel SDC build', separate=True)
     conda_build_cmd = ' '.join([

buildscripts/run_examples.py

@@ -90,6 +90,7 @@ def run_examples(sdc_utils):
     sdc_utils.log_info('Run Intel(R) SDC examples', separate=True)
     sdc_utils.log_info(sdc_utils.line_double)
     sdc_utils.create_environment()
-    sdc_utils.install_conda_package(['sdc'])
+    sdc_package = f'sdc={sdc_utils.get_sdc_version_from_channel()}'
+    sdc_utils.install_conda_package([sdc_package])
 
     run_examples(sdc_utils)

buildscripts/sdc-conda-recipe/meta.yaml

@@ -1,6 +1,6 @@
-{% set NUMBA_VERSION = "==0.48" %}
+{% set NUMBA_VERSION = "==0.49.1" %}
 {% set PANDAS_VERSION = "==0.25.3" %}
-{% set PYARROW_VERSION = "==0.15.1" %}
+{% set PYARROW_VERSION = "==0.17.0" %}
 
 package:
   name: sdc

buildscripts/utilities.py

@@ -26,6 +26,7 @@
 
 
 
+import json
 import os
 import platform
 import re
@@ -51,7 +52,7 @@ def __init__(self, python, sdc_local_channel=None):
         self.line_single = '-'*80
 
         # Set channels
-        self.channel_list = ['-c', 'intel/label/beta', '-c', 'intel', '-c', 'defaults', '-c', 'conda-forge']
+        self.channel_list = ['-c', 'intel/label/beta', '-c', 'defaults', '-c', 'conda-forge']
         if sdc_local_channel:
             sdc_local_channel = Path(sdc_local_channel).resolve().as_uri()
             self.channel_list = ['-c', sdc_local_channel] + self.channel_list
@@ -87,7 +88,7 @@ def create_environment(self, packages_list=[]):
         # Create Intel SDC environment
         create_args = ['-q', '-y', '-n', self.env_name, f'python={self.python}']
         create_args += packages_list + self.channel_list + ['--override-channels']
-        self.__run_conda_command(Conda_Commands.CREATE, create_args)
+        self.log_info(self.__run_conda_command(Conda_Commands.CREATE, create_args))
 
         return
 
@@ -97,7 +98,7 @@ def install_conda_package(self, packages_list):
         self.log_info(f'Install {" ".join(packages_list)} to {self.env_name} conda environment')
         install_args = ['-n', self.env_name]
         install_args += self.channel_list + ['--override-channels', '-q', '-y'] + packages_list
-        self.__run_conda_command(Conda_Commands.INSTALL, install_args)
+        self.log_info(self.__run_conda_command(Conda_Commands.INSTALL, install_args))
 
         return
 
@@ -135,3 +136,19 @@ def log_info(self, msg, separate=False):
         if separate:
             print(f'{time.strftime("%d/%m/%Y %H:%M:%S")}: {self.line_double}', flush=True)
         print(f'{time.strftime("%d/%m/%Y %H:%M:%S")}: {msg}', flush=True)
+
+    def get_sdc_version_from_channel(self):
+        python_version = 'py' + self.python.replace('.', '')
+
+        # Get Intel SDC version from first channel in channel_list
+        search_args = ['sdc', '-c', self.channel_list[1], '--override-channels', '--json']
+        search_result = self.__run_conda_command(Conda_Commands.SEARCH, search_args)
+
+        repo_data = json.loads(search_result)
+        for package_data in repo_data['sdc']:
+            sdc_version = package_data['version']
+            sdc_build = package_data['build']
+            if python_version in sdc_build:
+                break
+
+        return f'{sdc_version}={sdc_build}'

docs/source/compilation.rst

@@ -35,7 +35,7 @@ Output:
 
     Traceback (most recent call last):
     ...
-    numba.errors.TypingError: Failed in nopython mode pipeline (step: nopython frontend)
+    numba.core.errors.TypingError: Failed in nopython mode pipeline (step: nopython frontend)
     Unknown attribute 'read_excel' of type Module(<module 'pandas' from ...)
 
 

requirements.txt

@@ -1,4 +1,4 @@
 numpy>=1.16
 pandas==0.25.3
-pyarrow==0.15.1
-numba==0.48
+pyarrow==0.17.0
+numba==0.49.1

sdc/__init__.py

@@ -45,6 +45,8 @@
 import sdc.datatypes.hpat_pandas_stringmethods_functions
 import sdc.datatypes.hpat_pandas_groupby_functions
 
+import sdc.extensions.indexes.range_index_ext
+
 from ._version import get_versions
 
 if not sdc.config.config_pipeline_hpat_default:
@@ -57,8 +59,8 @@
     """
 
     # sdc.config.numba_compiler_define_nopython_pipeline_orig = \
-    # numba.compiler.DefaultPassBuilder.define_nopython_pipeline
-    # numba.compiler.DefaultPassBuilder.define_nopython_pipeline = \
+    # numba.core.compiler.DefaultPassBuilder.define_nopython_pipeline
+    # numba.core.compiler.DefaultPassBuilder.define_nopython_pipeline = \
     # sdc.datatypes.hpat_pandas_dataframe_pass.sdc_nopython_pipeline_lite_register
 
     import sdc.rewrites.dataframe_constructor

sdc/compiler.py

@@ -34,18 +34,18 @@
 from sdc.hiframes.hiframes_typed import HiFramesTypedPass
 from sdc.hiframes.dataframe_pass import DataFramePass
 import numba
-import numba.compiler
-from numba.compiler import DefaultPassBuilder
+import numba.core.compiler
+from numba.core.compiler import DefaultPassBuilder
 from numba import ir_utils, ir, postproc
-from numba.targets.registry import CPUDispatcher
-from numba.ir_utils import guard, get_definition
-from numba.inline_closurecall import inline_closure_call
+from numba.core.registry import CPUDispatcher
+from numba.core.ir_utils import guard, get_definition
+from numba.core.inline_closurecall import inline_closure_call
 from numba.typed_passes import (NopythonTypeInference, AnnotateTypes, ParforPass, IRLegalization)
 from numba.untyped_passes import (DeadBranchPrune, InlineInlinables, InlineClosureLikes)
 from sdc import config
 from sdc.distributed import DistributedPass
 
-from numba.compiler_machinery import FunctionPass, register_pass
+from numba.core.compiler_machinery import FunctionPass, register_pass
 
 # workaround for Numba #3876 issue with large labels in mortgage benchmark
 binding.set_option("tmp", "-non-global-value-max-name-size=2048")
@@ -143,7 +143,7 @@ def run_pass(self, state):
         return True
 
 
-class SDCPipeline(numba.compiler.CompilerBase):
+class SDCPipeline(numba.core.compiler.CompilerBase):
     """SDC compiler pipeline
     """
 
@@ -170,7 +170,7 @@ def __init__(self):
         pass
 
     def run_pass(self, state):
-        numba.parfor.lower_parfor_sequential(
+        numba.parfors.parfor.lower_parfor_sequential(
             state.typingctx, state.func_ir, state.typemap, state.calltypes)
 
         return True

sdc/cv_ext.py

@@ -28,12 +28,12 @@
 import numba
 import sdc
 from numba import types
-from numba.typing.templates import infer_global, AbstractTemplate, infer, signature
+from numba.core.typing.templates import infer_global, AbstractTemplate, infer, signature
 from numba.extending import lower_builtin, overload, intrinsic
-from numba import cgutils
+from numba.core import cgutils
 from sdc.str_ext import string_type
-from numba.targets.imputils import impl_ret_new_ref, impl_ret_borrowed
-from numba.targets.arrayobj import _empty_nd_impl
+from numba.core.imputils import impl_ret_new_ref, impl_ret_borrowed
+from numba.np.arrayobj import _empty_nd_impl
 
 import cv2
 import numpy as np