Skip to content

Commit 8e9cb87

Browse files
committed
ARROW-3602: [Gandiva] [Python] Initial Gandiva Cython bindings
This is an initial Cython wrapper for Gandiva. Feedback is appreciated (the API is experimental right now and will most likely change in the future). Author: Philipp Moritz <pcmoritz@gmail.com> Closes #2822 from pcmoritz/gandiva-cython and squashes the following commits: 6ff0d94 <Philipp Moritz> Merge branch 'master' into gandiva-cython 6ea0062 <Philipp Moritz> use cython instantiation 4a366bb <Philipp Moritz> Merge branch 'master' into gandiva-cython 73bc203 <Philipp Moritz> use strings instead of bytes for function names f589de3 <Philipp Moritz> fix travis 69b10cb <Philipp Moritz> don't build gandiva in mac os build 20b75a6 <Philipp Moritz> update e1d074b <Philipp Moritz> build gandiva tests 562b763 <Philipp Moritz> put back llvm 9b9d3ab <Philipp Moritz> don't build gandiva tests in python build 1ee6019 <Philipp Moritz> don't run gandiva tests on python build 646f360 <Philipp Moritz> Merge branch 'gandiva-cython' of github.com:pcmoritz/arrow into gandiva-cython 40bb0c7 <Philipp Moritz> use gandiva files cd282a3 <Philipp Moritz> Merge branch 'master' into gandiva-cython 829f7a2 <Philipp Moritz> fix f726d17 <Philipp Moritz> remove compiler error 99f93f1 <Philipp Moritz> Merge branch 'master' into gandiva-cython cfec265 <Philipp Moritz> use clang 6008884 <Philipp Moritz> install llvm 6.0 (?) 5abd248 <Philipp Moritz> build gandiva 9ee2c58 <Philipp Moritz> add gandiva flag 0380846 <Philipp Moritz> fix pytest include 27434d5 <Philipp Moritz> add ganvida pytest flags 8374cdb <Philipp Moritz> make gandiva optional for tests 5ceb221 <Philipp Moritz> allow gandiva failure 021b301 <Philipp Moritz> lint 276536c <Philipp Moritz> Merge branch 'master' into gandiva-cython b415994 <Philipp Moritz> remove gandiva cython bindings from wheels 7b75dec <Philipp Moritz> linting dc2a648 <Philipp Moritz> add filter 0ff33c1 <Philipp Moritz> add test b4571ee <Philipp Moritz> build gandiva for wheels 92c8dff <Philipp Moritz> linting ae5305a <Philipp Moritz> linting 97568d9 <Philipp Moritz> whitespace 335be39 <Philipp Moritz> update FindGandiva.cmake 33786d2 <Philipp Moritz> memory pool handling 26be866 <Philipp Moritz> fix array creation aab1770 <Philipp Moritz> fixes 7a8c948 <Philipp Moritz> port gandiva cython wrappers to in-tree gandiva
1 parent 3ab4a0f commit 8e9cb87

File tree

12 files changed

+590
-25
lines changed

12 files changed

+590
-25
lines changed

.travis.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,13 +78,15 @@ matrix:
7878
- ARROW_BUILD_WARNING_LEVEL=CHECKIN
7979
- ARROW_TRAVIS_PYTHON_JVM=1
8080
- ARROW_TRAVIS_JAVA_BUILD_ONLY=1
81+
- ARROW_TRAVIS_PYTHON_GANDIVA=1
8182
# ARROW-2999 Benchmarks are disabled in Travis CI for the time being
8283
# - ARROW_TRAVIS_PYTHON_BENCHMARKS=1
8384
- MATRIX_EVAL="CC=gcc-4.9 && CXX=g++-4.9"
8485
before_script:
8586
# (ARROW_CI_CPP_AFFECTED implies ARROW_CI_PYTHON_AFFECTED)
8687
- if [ $ARROW_CI_PYTHON_AFFECTED != "1" ]; then exit; fi
8788
- $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
89+
- $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
8890
# If either C++ or Python changed, we must install the C++ libraries
8991
- git submodule update --init
9092
- $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh

ci/travis_script_python.sh

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,10 @@ if [ $ARROW_TRAVIS_COVERAGE == "1" ]; then
9696
CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_GENERATE_COVERAGE=ON"
9797
fi
9898

99+
if [ $ARROW_TRAVIS_PYTHON_GANDIVA == "1" ]; then
100+
CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_GANDIVA=ON -DARROW_GANDIVA_BUILD_TESTS=OFF"
101+
fi
102+
99103
cmake -GNinja \
100104
$CMAKE_COMMON_FLAGS \
101105
-DARROW_BUILD_TESTS=on \
@@ -136,6 +140,9 @@ export PYARROW_BUILD_TYPE=$ARROW_BUILD_TYPE
136140
export PYARROW_WITH_PARQUET=1
137141
export PYARROW_WITH_PLASMA=1
138142
export PYARROW_WITH_ORC=1
143+
if [ $ARROW_TRAVIS_PYTHON_GANDIVA == "1" ]; then
144+
export PYARROW_WITH_GANDIVA=1
145+
fi
139146

140147
python setup.py develop
141148

@@ -201,6 +208,7 @@ if [ "$ARROW_TRAVIS_PYTHON_BENCHMARKS" == "1" ] && [ "$PYTHON_VERSION" == "3.6"
201208
export PYARROW_WITH_PARQUET=1
202209
export PYARROW_WITH_PLASMA=1
203210
export PYARROW_WITH_ORC=0
211+
export PYARROW_WITH_GANDIVA=0
204212

205213
pushd $ARROW_PYTHON_DIR
206214
# Workaround for https://github.com/airspeed-velocity/asv/issues/631

cpp/CMakeLists.txt

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -317,6 +317,10 @@ Always OFF if building binaries"
317317
"Build the Gandiva JNI wrappers"
318318
ON)
319319

320+
option(ARROW_GANDIVA_BUILD_TESTS
321+
"Build the Gandiva googletest unit tests"
322+
ON)
323+
320324
endif()
321325

322326

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
# - Find GANDIVA (gandiva/client.h, libgandiva.a, libgandiva.so)
19+
# This module defines
20+
# GANDIVA_INCLUDE_DIR, directory containing headers
21+
# GANDIVA_LIBS, directory containing gandiva libraries
22+
# GANDIVA_STATIC_LIB, path to libgandiva.a
23+
# GANDIVA_SHARED_LIB, path to libgandiva's shared library
24+
# GANDIVA_SHARED_IMP_LIB, path to libgandiva's import library (MSVC only)
25+
# GANDIVA_FOUND, whether gandiva has been found
26+
27+
include(FindPkgConfig)
28+
29+
if ("$ENV{ARROW_HOME}" STREQUAL "")
30+
pkg_check_modules(GANDIVA gandiva)
31+
if (GANDIVA_FOUND)
32+
pkg_get_variable(GANDIVA_SO_VERSION gandiva so_version)
33+
set(GANDIVA_ABI_VERSION ${GANDIVA_SO_VERSION})
34+
message(STATUS "Gandiva SO and ABI version: ${GANDIVA_SO_VERSION}")
35+
pkg_get_variable(GANDIVA_FULL_SO_VERSION gandiva full_so_version)
36+
message(STATUS "Gandiva full SO version: ${GANDIVA_FULL_SO_VERSION}")
37+
set(GANDIVA_INCLUDE_DIR ${GANDIVA_INCLUDE_DIRS})
38+
set(GANDIVA_LIBS ${GANDIVA_LIBRARY_DIRS})
39+
set(GANDIVA_SEARCH_LIB_PATH ${GANDIVA_LIBRARY_DIRS})
40+
endif()
41+
else()
42+
set(GANDIVA_HOME "$ENV{ARROW_HOME}")
43+
44+
set(GANDIVA_SEARCH_HEADER_PATHS
45+
${GANDIVA_HOME}/include
46+
)
47+
48+
set(GANDIVA_SEARCH_LIB_PATH
49+
${GANDIVA_HOME}/lib
50+
)
51+
52+
find_path(GANDIVA_INCLUDE_DIR gandiva/expression_registry.h PATHS
53+
${GANDIVA_SEARCH_HEADER_PATHS}
54+
# make sure we don't accidentally pick up a different version
55+
NO_DEFAULT_PATH
56+
)
57+
endif()
58+
59+
find_library(GANDIVA_LIB_PATH NAMES gandiva
60+
PATHS
61+
${GANDIVA_SEARCH_LIB_PATH}
62+
NO_DEFAULT_PATH)
63+
get_filename_component(GANDIVA_LIBS ${GANDIVA_LIB_PATH} DIRECTORY)
64+
65+
if (GANDIVA_INCLUDE_DIR AND GANDIVA_LIBS)
66+
set(GANDIVA_FOUND TRUE)
67+
set(GANDIVA_LIB_NAME gandiva)
68+
69+
set(GANDIVA_STATIC_LIB ${GANDIVA_LIBS}/lib${GANDIVA_LIB_NAME}.a)
70+
71+
set(GANDIVA_SHARED_LIB ${GANDIVA_LIBS}/lib${GANDIVA_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
72+
endif()
73+
74+
if (GANDIVA_FOUND)
75+
if (NOT Gandiva_FIND_QUIETLY)
76+
message(STATUS "Found the Gandiva core library: ${GANDIVA_LIB_PATH}")
77+
endif ()
78+
else ()
79+
if (NOT Gandiva_FIND_QUIETLY)
80+
set(GANDIVA_ERR_MSG "Could not find the Gandiva library. Looked for headers")
81+
set(GANDIVA_ERR_MSG "${GANDIVA_ERR_MSG} in ${GANDIVA_SEARCH_HEADER_PATHS}, and for libs")
82+
set(GANDIVA_ERR_MSG "${GANDIVA_ERR_MSG} in ${GANDIVA_SEARCH_LIB_PATH}")
83+
if (Gandiva_FIND_REQUIRED)
84+
message(FATAL_ERROR "${GANDIVA_ERR_MSG}")
85+
else (Gandiva_FIND_REQUIRED)
86+
message(STATUS "${GANDIVA_ERR_MSG}")
87+
endif (Gandiva_FIND_REQUIRED)
88+
endif ()
89+
set(GANDIVA_FOUND FALSE)
90+
endif ()
91+
92+
mark_as_advanced(
93+
GANDIVA_INCLUDE_DIR
94+
GANDIVA_STATIC_LIB
95+
GANDIVA_SHARED_LIB
96+
)

cpp/src/gandiva/CMakeLists.txt

Lines changed: 28 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -120,34 +120,38 @@ install(
120120
FILES "${CMAKE_CURRENT_BINARY_DIR}/gandiva.pc"
121121
DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig/")
122122

123-
#args: label test-file src-files
124-
add_gandiva_unit_test(bitmap_accumulator_test.cc bitmap_accumulator.cc)
125-
add_gandiva_unit_test(engine_llvm_test.cc engine.cc llvm_types.cc configuration.cc
126-
gdv_function_stubs.cc context_helper.cc to_date_holder.cc date_utils.cc
127-
exported_funcs_registry.cc ${BC_FILE_PATH_CC})
128-
add_gandiva_unit_test(function_signature_test.cc function_signature.cc)
129-
add_gandiva_unit_test(function_registry_test.cc function_registry.cc function_signature.cc)
130-
add_gandiva_unit_test(llvm_types_test.cc llvm_types.cc)
131-
add_gandiva_unit_test(llvm_generator_test.cc llvm_generator.cc regex_util.cc engine.cc
132-
llvm_types.cc expr_decomposer.cc function_registry.cc annotator.cc
133-
bitmap_accumulator.cc configuration.cc function_signature.cc like_holder.cc
134-
to_date_holder.cc date_utils.cc regex_util.cc gdv_function_stubs.cc context_helper.cc
135-
exported_funcs_registry.cc ${BC_FILE_PATH_CC})
136-
add_gandiva_unit_test(annotator_test.cc annotator.cc function_signature.cc)
137-
add_gandiva_unit_test(tree_expr_test.cc tree_expr_builder.cc expr_decomposer.cc annotator.cc function_registry.cc function_signature.cc like_holder.cc regex_util.cc to_date_holder.cc date_utils.cc)
138-
add_gandiva_unit_test(expr_decomposer_test.cc expr_decomposer.cc tree_expr_builder.cc annotator.cc function_registry.cc function_signature.cc like_holder.cc regex_util.cc to_date_holder.cc date_utils.cc)
139-
add_gandiva_unit_test(expression_registry_test.cc llvm_types.cc expression_registry.cc function_signature.cc function_registry.cc)
140-
add_gandiva_unit_test(selection_vector_test.cc selection_vector.cc)
141-
add_gandiva_unit_test(lru_cache_test.cc)
142-
add_gandiva_unit_test(to_date_holder_test.cc to_date_holder.cc date_utils.cc)
143-
add_gandiva_unit_test(simple_arena_test.cc)
123+
if (ARROW_GANDIVA_BUILD_TESTS)
124+
#args: label test-file src-files
125+
add_gandiva_unit_test(bitmap_accumulator_test.cc bitmap_accumulator.cc)
126+
add_gandiva_unit_test(engine_llvm_test.cc engine.cc llvm_types.cc configuration.cc
127+
gdv_function_stubs.cc context_helper.cc to_date_holder.cc date_utils.cc
128+
exported_funcs_registry.cc ${BC_FILE_PATH_CC})
129+
add_gandiva_unit_test(function_signature_test.cc function_signature.cc)
130+
add_gandiva_unit_test(function_registry_test.cc function_registry.cc function_signature.cc)
131+
add_gandiva_unit_test(llvm_types_test.cc llvm_types.cc)
132+
add_gandiva_unit_test(llvm_generator_test.cc llvm_generator.cc regex_util.cc engine.cc
133+
llvm_types.cc expr_decomposer.cc function_registry.cc annotator.cc
134+
bitmap_accumulator.cc configuration.cc function_signature.cc like_holder.cc
135+
to_date_holder.cc date_utils.cc regex_util.cc gdv_function_stubs.cc context_helper.cc
136+
exported_funcs_registry.cc ${BC_FILE_PATH_CC})
137+
add_gandiva_unit_test(annotator_test.cc annotator.cc function_signature.cc)
138+
add_gandiva_unit_test(tree_expr_test.cc tree_expr_builder.cc expr_decomposer.cc annotator.cc function_registry.cc function_signature.cc like_holder.cc regex_util.cc to_date_holder.cc date_utils.cc)
139+
add_gandiva_unit_test(expr_decomposer_test.cc expr_decomposer.cc tree_expr_builder.cc annotator.cc function_registry.cc function_signature.cc like_holder.cc regex_util.cc to_date_holder.cc date_utils.cc)
140+
add_gandiva_unit_test(expression_registry_test.cc llvm_types.cc expression_registry.cc function_signature.cc function_registry.cc)
141+
add_gandiva_unit_test(selection_vector_test.cc selection_vector.cc)
142+
add_gandiva_unit_test(lru_cache_test.cc)
143+
add_gandiva_unit_test(to_date_holder_test.cc to_date_holder.cc date_utils.cc)
144+
add_gandiva_unit_test(simple_arena_test.cc)
145+
endif()
144146

145147
if (ARROW_GANDIVA_JAVA)
146148
add_subdirectory(jni)
147149
endif()
148150
add_subdirectory(precompiled)
149151

150-
include(CTest)
151-
enable_testing()
152+
if (ARROW_GANDIVA_BUILD_TESTS)
153+
include(CTest)
154+
enable_testing()
152155

153-
add_subdirectory(tests)
156+
add_subdirectory(tests)
157+
endif()

cpp/src/gandiva/tests/generate_data.h

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,6 @@ class BoundedInt32DataGenerator : public Int32DataGenerator {
6767

6868
protected:
6969
uint32_t upperBound_;
70-
Random random_;
7170
};
7271

7372
class Int64DataGenerator : public DataGenerator<int64_t> {

python/CMakeLists.txt

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -488,6 +488,30 @@ if (PYARROW_BUILD_ORC)
488488
_orc)
489489
endif()
490490

491+
## Gandiva
492+
if (PYARROW_BUILD_GANDIVA)
493+
find_package(Gandiva)
494+
495+
if(NOT GANDIVA_FOUND)
496+
message(FATAL_ERROR "Unable to locate Gandiva libraries")
497+
endif()
498+
499+
include_directories(SYSTEM ${GANDIVA_INCLUDE_DIR})
500+
501+
if (PYARROW_BUNDLE_ARROW_CPP)
502+
bundle_arrow_lib(GANDIVA_SHARED_LIB
503+
ABI_VERSION ${ARROW_ABI_VERSION}
504+
SO_VERSION ${ARROW_SO_VERSION})
505+
endif()
506+
507+
set(LINK_LIBS
508+
${LINK_LIBS}
509+
${GANDIVA_SHARED_LIB})
510+
511+
set(CYTHON_EXTENSIONS ${CYTHON_EXTENSIONS} gandiva)
512+
endif()
513+
514+
491515
############################################################
492516
# Setup and build Cython modules
493517
############################################################

0 commit comments

Comments
 (0)