Skip to content

Commit 7a8c948

Browse files
pcmoritzpcmoritz
committed
port gandiva cython wrappers to in-tree gandiva
1 parent 0387ebc commit 7a8c948

File tree

7 files changed

+394
-1
lines changed

7 files changed

+394
-1
lines changed

‎cpp/cmake_modules/FindGandiva.cmake‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
# - Find GANDIVA (gandiva/client.h, libgandiva.a, libgandiva.so)
19+
# This module defines
20+
# GANDIVA_INCLUDE_DIR, directory containing headers
21+
# GANDIVA_LIBS, directory containing gandiva libraries
22+
# GANDIVA_STATIC_LIB, path to libgandiva.a
23+
# GANDIVA_SHARED_LIB, path to libgandiva's shared library
24+
# GANDIVA_SHARED_IMP_LIB, path to libgandiva's import library (MSVC only)
25+
# GANDIVA_FOUND, whether gandiva has been found
26+
27+
include(FindPkgConfig)
28+
29+
if ("$ENV{ARROW_HOME}" STREQUAL "")
30+
pkg_check_modules(GANDIVA gandiva)
31+
if (GANDIVA_FOUND)
32+
pkg_get_variable(GANDIVA_EXECUTABLE gandiva executable)
33+
pkg_get_variable(GANDIVA_SO_VERSION gandiva so_version)
34+
set(GANDIVA_ABI_VERSION ${GANDIVA_SO_VERSION})
35+
message(STATUS "Gandiva SO and ABI version: ${GANDIVA_SO_VERSION}")
36+
pkg_get_variable(GANDIVA_FULL_SO_VERSION gandiva full_so_version)
37+
message(STATUS "Gandiva full SO version: ${GANDIVA_FULL_SO_VERSION}")
38+
set(GANDIVA_INCLUDE_DIR ${GANDIVA_INCLUDE_DIRS})
39+
set(GANDIVA_LIBS ${GANDIVA_LIBRARY_DIRS})
40+
set(GANDIVA_SEARCH_LIB_PATH ${GANDIVA_LIBRARY_DIRS})
41+
endif()
42+
else()
43+
set(GANDIVA_HOME "$ENV{ARROW_HOME}")
44+
45+
set(GANDIVA_EXECUTABLE ${GANDIVA_HOME}/bin/gandiva_store)
46+
47+
set(GANDIVA_SEARCH_HEADER_PATHS
48+
${GANDIVA_HOME}/include
49+
)
50+
51+
set(GANDIVA_SEARCH_LIB_PATH
52+
${GANDIVA_HOME}/lib
53+
)
54+
55+
find_path(GANDIVA_INCLUDE_DIR gandiva/client.h PATHS
56+
${GANDIVA_SEARCH_HEADER_PATHS}
57+
# make sure we don't accidentally pick up a different version
58+
NO_DEFAULT_PATH
59+
)
60+
endif()
61+
62+
find_library(GANDIVA_LIB_PATH NAMES gandiva
63+
PATHS
64+
${GANDIVA_SEARCH_LIB_PATH}
65+
NO_DEFAULT_PATH)
66+
get_filename_component(GANDIVA_LIBS ${GANDIVA_LIB_PATH} DIRECTORY)
67+
68+
if (GANDIVA_INCLUDE_DIR AND GANDIVA_LIBS)
69+
set(GANDIVA_FOUND TRUE)
70+
set(GANDIVA_LIB_NAME gandiva)
71+
72+
set(GANDIVA_STATIC_LIB ${GANDIVA_LIBS}/lib${GANDIVA_LIB_NAME}.a)
73+
74+
set(GANDIVA_SHARED_LIB ${GANDIVA_LIBS}/lib${GANDIVA_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
75+
endif()
76+
77+
if (GANDIVA_FOUND)
78+
if (NOT Gandiva_FIND_QUIETLY)
79+
message(STATUS "Found the Gandiva core library: ${GANDIVA_LIB_PATH}")
80+
message(STATUS "Found Gandiva executable: ${GANDIVA_EXECUTABLE}")
81+
endif ()
82+
else ()
83+
if (NOT Gandiva_FIND_QUIETLY)
84+
set(GANDIVA_ERR_MSG "Could not find the Gandiva library. Looked for headers")
85+
set(GANDIVA_ERR_MSG "${GANDIVA_ERR_MSG} in ${GANDIVA_SEARCH_HEADER_PATHS}, and for libs")
86+
set(GANDIVA_ERR_MSG "${GANDIVA_ERR_MSG} in ${GANDIVA_SEARCH_LIB_PATH}")
87+
if (Gandiva_FIND_REQUIRED)
88+
message(FATAL_ERROR "${GANDIVA_ERR_MSG}")
89+
else (Gandiva_FIND_REQUIRED)
90+
message(STATUS "${GANDIVA_ERR_MSG}")
91+
endif (Gandiva_FIND_REQUIRED)
92+
endif ()
93+
set(GANDIVA_FOUND FALSE)
94+
endif ()
95+
96+
mark_as_advanced(
97+
GANDIVA_INCLUDE_DIR
98+
GANDIVA_STATIC_LIB
99+
GANDIVA_SHARED_LIB
100+
)

‎cpp/src/gandiva/CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ install(FILES
115115
selection_vector.h
116116
status.h
117117
tree_expr_builder.h
118-
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
118+
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/gandiva
119119
)
120120

121121
# pkg-config support

‎python/CMakeLists.txt‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -488,6 +488,21 @@ if (PYARROW_BUILD_ORC)
488488
_orc)
489489
endif()
490490

491+
## Gandiva
492+
if (PYARROW_BUILD_GANDIVA)
493+
find_package(Gandiva)
494+
495+
if(NOT GANDIVA_FOUND)
496+
message(FATAL_ERROR "Unable to locate Gandiva libraries")
497+
endif()
498+
499+
message("XXX" ${GANDIVA_INCLUDE_DIR})
500+
include_directories(SYSTEM ${GANDIVA_INCLUDE_DIR})
501+
502+
set(CYTHON_EXTENSIONS ${CYTHON_EXTENSIONS} gandiva)
503+
endif()
504+
505+
491506
############################################################
492507
# Setup and build Cython modules
493508
############################################################

‎python/pyarrow/gandiva.pyx‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
# cython: profile=False
19+
# distutils: language = c++
20+
# cython: embedsignature = True
21+
22+
from libcpp cimport bool as c_bool, nullptr
23+
from libcpp.memory cimport shared_ptr, unique_ptr, make_shared
24+
from libcpp.string cimport string as c_string
25+
from libcpp.vector cimport vector as c_vector
26+
from libc.stdint cimport int64_t, uint8_t, uintptr_t
27+
28+
from pyarrow.includes.libarrow cimport *
29+
from pyarrow.compat import frombytes
30+
31+
from pyarrow.includes.libgandiva cimport (GStatus, CExpression,
32+
CNode, CProjector,
33+
TreeExprBuilder_MakeExpression,
34+
TreeExprBuilder_MakeFunction,
35+
TreeExprBuilder_MakeLiteral,
36+
TreeExprBuilder_MakeField,
37+
TreeExprBuilder_MakeIf,
38+
Projector_Make)
39+
40+
from pyarrow.lib cimport (Array, DataType, Field, MemoryPool,
41+
RecordBatch, Schema)
42+
43+
cdef int check_status(const GStatus& status) nogil except -1:
44+
if status.ok():
45+
return 0
46+
47+
with gil:
48+
message = frombytes(status.message())
49+
raise Exception(message)
50+
51+
cdef class Node:
52+
cdef:
53+
shared_ptr[CNode] node
54+
55+
cdef void init(self, shared_ptr[CNode] node):
56+
self.node = node
57+
58+
cdef make_node(shared_ptr[CNode] node):
59+
cdef Node result = Node()
60+
result.init(node)
61+
return result
62+
63+
cdef class Expression:
64+
cdef:
65+
shared_ptr[CExpression] expression
66+
67+
cdef void init(self, shared_ptr[CExpression] expression):
68+
self.expression = expression
69+
70+
cdef make_array(shared_ptr[CArray] array):
71+
cdef Array result = Array()
72+
result.init(array)
73+
return result
74+
75+
cdef class Projector:
76+
cdef:
77+
shared_ptr[CProjector] projector
78+
79+
cdef void init(self, shared_ptr[CProjector] projector):
80+
self.projector = projector
81+
82+
def evaluate(self, RecordBatch batch):
83+
cdef vector[shared_ptr[CArray]] results
84+
check_status(self.projector.get().Evaluate(
85+
batch.sp_batch.get()[0], c_get_memory_pool(), &results))
86+
cdef shared_ptr[CArray] result
87+
arrays = []
88+
for result in results:
89+
arrays.append(make_array(result))
90+
return arrays
91+
92+
cdef class TreeExprBuilder:
93+
94+
def make_literal(self, value):
95+
cdef shared_ptr[CNode] r = TreeExprBuilder_MakeLiteral(value)
96+
return make_node(r)
97+
98+
def make_expression(self, Node root_node, Field return_field):
99+
cdef shared_ptr[CExpression] r = TreeExprBuilder_MakeExpression(
100+
root_node.node, return_field.sp_field)
101+
cdef Expression expression = Expression()
102+
expression.init(r)
103+
return expression
104+
105+
def make_function(self, name, children, DataType return_type):
106+
cdef c_vector[shared_ptr[CNode]] c_children
107+
cdef Node child
108+
for child in children:
109+
c_children.push_back(child.node)
110+
cdef shared_ptr[CNode] r = TreeExprBuilder_MakeFunction(
111+
name, c_children, return_type.sp_type)
112+
return make_node(r)
113+
114+
def make_field(self, Field field):
115+
cdef shared_ptr[CNode] r = TreeExprBuilder_MakeField(field.sp_field)
116+
return make_node(r)
117+
118+
def make_if(self, Node condition, Node this_node,
119+
Node else_node, DataType return_type):
120+
cdef shared_ptr[CNode] r = TreeExprBuilder_MakeIf(
121+
condition.node, this_node.node, else_node.node,
122+
return_type.sp_type)
123+
return make_node(r)
124+
125+
cpdef make_projector(Schema schema, children, MemoryPool pool):
126+
cdef c_vector[shared_ptr[CExpression]] c_children
127+
cdef Expression child
128+
for child in children:
129+
c_children.push_back(child.expression)
130+
cdef shared_ptr[CProjector] result
131+
check_status(Projector_Make(schema.sp_schema, c_children,
132+
&result))
133+
cdef Projector projector = Projector()
134+
projector.init(result)
135+
return projector

‎python/pyarrow/includes/libgandiva.pxd‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
# distutils: language = c++
19+
20+
from pyarrow.includes.common cimport *
21+
from pyarrow.includes.libarrow cimport *
22+
23+
cdef extern from "gandiva/status.h" namespace "gandiva" nogil:
24+
# We can later add more of the common status factory methods as needed
25+
cdef GStatus GStatus_OK "Status::OK"()
26+
cdef GStatus GStatus_Invalid "Status::Invalid"()
27+
28+
cdef cppclass GStatus "gandiva::Status":
29+
GStatus()
30+
31+
c_string ToString()
32+
c_string message()
33+
34+
c_bool ok()
35+
36+
cdef extern from "gandiva/gandiva_aliases.h" namespace "gandiva" nogil:
37+
38+
cdef cppclass CNode" gandiva::Node":
39+
pass
40+
41+
cdef cppclass CExpression" gandiva::Expression":
42+
pass
43+
44+
ctypedef vector[shared_ptr[CNode]] CNodeVector" gandiva::NodeVector"
45+
46+
ctypedef vector[shared_ptr[CExpression]] \
47+
CExpressionVector" gandiva::ExpressionVector"
48+
49+
50+
cdef extern from "gandiva/arrow.h" namespace "gandiva" nogil:
51+
52+
ctypedef vector[shared_ptr[CArray]] CArrayVector" gandiva::ArrayVector"
53+
54+
55+
cdef extern from "gandiva/tree_expr_builder.h" namespace "gandiva" nogil:
56+
57+
cdef shared_ptr[CNode] TreeExprBuilder_MakeLiteral \
58+
"gandiva::TreeExprBuilder::MakeLiteral"(c_bool value)
59+
60+
cdef shared_ptr[CExpression] TreeExprBuilder_MakeExpression\
61+
"gandiva::TreeExprBuilder::MakeExpression"(
62+
shared_ptr[CNode] root_node, shared_ptr[CField] result_field)
63+
64+
cdef shared_ptr[CNode] TreeExprBuilder_MakeFunction \
65+
"gandiva::TreeExprBuilder::MakeFunction"(
66+
const c_string& name, const CNodeVector& children,
67+
shared_ptr[CDataType] return_type)
68+
69+
cdef shared_ptr[CNode] TreeExprBuilder_MakeField \
70+
"gandiva::TreeExprBuilder::MakeField"(shared_ptr[CField] field)
71+
72+
cdef shared_ptr[CNode] TreeExprBuilder_MakeIf \
73+
"gandiva::TreeExprBuilder::MakeIf"(
74+
shared_ptr[CNode] condition, shared_ptr[CNode] this_node,
75+
shared_ptr[CNode] else_node, shared_ptr[CDataType] return_type)
76+
77+
cdef GStatus Projector_Make \
78+
"gandiva::Projector::Make"(
79+
shared_ptr[CSchema] schema, const CExpressionVector& children,
80+
shared_ptr[CProjector]* projector)
81+
82+
cdef extern from "gandiva/projector.h" namespace "gandiva" nogil:
83+
84+
cdef cppclass CProjector" gandiva::Projector":
85+
86+
GStatus Evaluate(const CRecordBatch& batch, CMemoryPool* pool, const CArrayVector* output)

0 commit comments

Comments
 (0)