Skip to content

Commit 29f76df

Browse files
wesmwesm
committed
ARROW-4446: [C++][Python] Run Gandiva C++ unit tests in Appveyor, get build and tests working in Python
Resolves * ARROW-4014 Fix warning about LIBCMT in MSVC build * ARROW-4445 Run Gandiva C++ tests in Appveyor * ARROW-4446 Run Gandiva Python unit tests in Appveyor * ARROW-4404 Appveyor toolchain build does not actually build the project Author: Wes McKinney <wesm+git@apache.org> Closes #3567 from wesm/gandiva-python-windows and squashes the following commits: d9e4b1b <Wes McKinney> Only use /nodefaultlib:libcmt if arrow_use_static_crt=OFF a8f960d <Wes McKinney> Fix appyveyor.yml 72c97e2 <Wes McKinney> Re-enable all Appyveyor entries a2e57d1 <Wes McKinney> Prevent move_shared_libs from clobbering the gandiva.cpp file which downstream code is expecting to find d78ae13 <Wes McKinney> Debug prints ac402f0 <Wes McKinney> Set language_level = 3 8ae63f1 <Wes McKinney> Fix LLVM 7 issues with MSVC, enable FindClangTools.cmake to find clang-format in conda env 3677115 <Wes McKinney> Do not do verbose linking 773c731 <Wes McKinney> Fixes 00d4126 <Wes McKinney> Disable failing Windows tests 063543c <Wes McKinney> Don't touch linker flags unless on MSVC e19b688 <Wes McKinney> Upgrade to LLVM 7 707bc49 <Wes McKinney> Fix for gflags 5897045 <Wes McKinney> Retool FindGFlags.cmake a bit to try to fix Windows toolchain build c2bb420 <Wes McKinney> Install all conda toolchain packages in one go c285776 <Wes McKinney> Rearrange conda calls to see if fixes build 469a2df <Wes McKinney> Enable gandiva is one build entry b05cf08 <Wes McKinney> Get Python build and tests for Gandiva working on, add to Appveyor build
1 parent 27ff2b3 commit 29f76df

File tree

16 files changed

+163
-76
lines changed

16 files changed

+163
-76
lines changed

‎appveyor.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ environment:
4848
global:
4949
USE_CLCACHE: true
5050
ARROW_BUILD_GANDIVA: "OFF"
51+
ARROW_LLVM_VERSION: "7.0.*"
5152
PYTHON: "3.6"
5253
ARCH: "64"
5354

@@ -61,6 +62,7 @@ environment:
6162
- JOB: "Toolchain"
6263
GENERATOR: Visual Studio 14 2015 Win64
6364
CONFIGURATION: "Release"
65+
ARROW_BUILD_GANDIVA: "ON"
6466
- JOB: "Static_Crt_Build"
6567
GENERATOR: Ninja
6668
- JOB: "Build_Debug"

‎ci/appveyor-cpp-build.bat‎

Lines changed: 11 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -95,33 +95,24 @@ if "%JOB%" == "Build_Debug" (
9595
exit /B 0
9696
)
9797

98-
conda create -n arrow -q -y -c conda-forge ^
99-
--file=ci\conda_env_python.yml ^
100-
python=%PYTHON% ^
101-
numpy=1.14 ^
102-
thrift-cpp=0.11 ^
103-
boost-cpp
104-
105-
call activate arrow
106-
107-
set ARROW_LLVM_VERSION=6.0.1
98+
set CONDA_PACKAGES=--file=ci\conda_env_python.yml python=%PYTHON% numpy=1.14 thrift-cpp=0.11 boost-cpp
10899

109100
if "%ARROW_BUILD_GANDIVA%" == "ON" (
110101
@rem Install llvmdev in the toolchain if building gandiva.dll
111-
conda install -q -y llvmdev=%ARROW_LLVM_VERSION% || exit /B
102+
set CONDA_PACKAGES=%CONDA_PACKAGES% llvmdev=%ARROW_LLVM_VERSION% clangdev=%ARROW_LLVM_VERSION%
112103
)
113104

114-
@rem Use Boost from Anaconda
115-
set BOOST_ROOT=%CONDA_PREFIX%\Library
116-
set BOOST_LIBRARYDIR=%CONDA_PREFIX%\Library\lib
117-
118105
if "%JOB%" == "Toolchain" (
119106
@rem Install pre-built "toolchain" packages for faster builds
120-
conda install -q -y -c conda-forge ^
121-
--file=ci\conda_env_cpp.yml ^
122-
python=%PYTHON%
123-
124-
set ARROW_BUILD_TOOLCHAIN=%CONDA_PREFIX%\Library
107+
set CONDA_PACKAGES=%CONDA_PACKAGES% --file=ci\conda_env_cpp.yml
125108
)
126109

110+
conda create -n arrow -q -y %CONDA_PACKAGES% -c conda-forge
111+
112+
call activate arrow
113+
114+
@rem Use Boost from Anaconda
115+
set BOOST_ROOT=%CONDA_PREFIX%\Library
116+
set BOOST_LIBRARYDIR=%CONDA_PREFIX%\Library\lib
117+
127118
call ci\cpp-msvc-build-main.bat

‎ci/cpp-msvc-build-main.bat‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,8 @@ set ARROW_HOME=%CONDA_PREFIX%\Library
2222
set CMAKE_ARGS=-DARROW_VERBOSE_THIRDPARTY_BUILD=OFF
2323

2424
if "%JOB%" == "Toolchain" (
25-
set CMAKE_ARGS=%CMAKE_ARGS% -DARROW_WITH_BZ2=ON
25+
set CMAKE_ARGS=%CMAKE_ARGS% -DARROW_WITH_BZ2=ON
26+
set ARROW_BUILD_TOOLCHAIN=%CONDA_PREFIX%\Library
2627
)
2728

2829
@rem Retrieve git submodules, configure env var for Parquet unit tests
@@ -85,6 +86,7 @@ set PYARROW_BUNDLE_ARROW_CPP=ON
8586
set PYARROW_BUNDLE_BOOST=OFF
8687
set PYARROW_WITH_STATIC_BOOST=ON
8788
set PYARROW_WITH_PARQUET=ON
89+
set PYARROW_WITH_GANDIVA=%ARROW_BUILD_GANDIVA%
8890
set PYARROW_PARALLEL=2
8991

9092
@rem ARROW-3075; pkgconfig is broken for Parquet for now

‎cpp/CMakeLists.txt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -311,6 +311,10 @@ Note that this requires linking Boost statically"
311311
# Windows options
312312

313313
if (MSVC)
314+
option(MSVC_LINK_VERBOSE
315+
"Pass verbose linking options when linking libraries and executables"
316+
OFF)
317+
314318
option(ARROW_USE_CLCACHE
315319
"Use clcache if available"
316320
ON)

‎cpp/cmake_modules/FindClangTools.cmake‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,8 +53,10 @@ set(CLANG_TOOLS_SEARCH_PATHS
5353
${ClangTools_PATH}
5454
$ENV{CLANG_TOOLS_PATH}
5555
/usr/local/bin /usr/bin
56-
"C:/Program Files/LLVM/bin"
57-
"${HOMEBREW_PREFIX}/bin")
56+
"C:/Program Files/LLVM/bin" # Windows, non-conda
57+
"$ENV{CONDA_PREFIX}/Library/bin" # Windows, conda
58+
"${HOMEBREW_PREFIX}/bin"
59+
)
5860

5961
find_program(CLANG_TIDY_BIN
6062
NAMES clang-tidy-${ARROW_LLVM_VERSION}

‎cpp/cmake_modules/FindGFlags.cmake‎

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -39,16 +39,18 @@ set(GFLAGS_STATIC_LIB_SUFFIX
3939
set(GFLAGS_STATIC_LIB_NAME
4040
${CMAKE_STATIC_LIBRARY_PREFIX}gflags${GFLAGS_STATIC_LIB_SUFFIX})
4141

42+
message(STATUS "GFLAGS_HOME: ${GFLAGS_HOME}")
43+
4244
if ( _gflags_roots )
4345
find_path(GFLAGS_INCLUDE_DIR NAMES gflags/gflags.h
4446
PATHS ${_gflags_roots}
4547
NO_DEFAULT_PATH
4648
PATH_SUFFIXES "include" )
47-
find_library(GFLAGS_SHARED_LIB NAMES gflags
49+
find_library(GFLAGS_STATIC_LIB NAMES ${GFLAGS_STATIC_LIB_NAME}
4850
PATHS ${_gflags_roots}
4951
NO_DEFAULT_PATH
5052
PATH_SUFFIXES "lib" )
51-
find_library(GFLAGS_STATIC_LIB NAMES ${GFLAGS_STATIC_LIB_NAME}
53+
find_library(GFLAGS_SHARED_LIB NAMES gflags
5254
PATHS ${_gflags_roots}
5355
NO_DEFAULT_PATH
5456
PATH_SUFFIXES "lib" )
@@ -57,14 +59,21 @@ else()
5759
# make sure we don't accidentally pick up a different version
5860
NO_CMAKE_SYSTEM_PATH
5961
NO_SYSTEM_ENVIRONMENT_PATH)
60-
find_library(GFLAGS_SHARED_LIB gflags
62+
find_library(GFLAGS_STATIC_LIB ${GFLAGS_STATIC_LIB_NAME}
6163
NO_CMAKE_SYSTEM_PATH
6264
NO_SYSTEM_ENVIRONMENT_PATH)
63-
find_library(GFLAGS_STATIC_LIB ${GFLAGS_STATIC_LIB_NAME}
65+
find_library(GFLAGS_SHARED_LIB gflags
6466
NO_CMAKE_SYSTEM_PATH
6567
NO_SYSTEM_ENVIRONMENT_PATH)
6668
endif()
6769

68-
include(FindPackageHandleStandardArgs)
69-
find_package_handle_standard_args(GFLAGS REQUIRED_VARS
70-
GFLAGS_SHARED_LIB GFLAGS_STATIC_LIB GFLAGS_INCLUDE_DIR)
70+
if (GFLAGS_INCLUDE_DIR AND GFLAGS_STATIC_LIB)
71+
set(GFLAGS_FOUND TRUE)
72+
else ()
73+
set(GFLAGS_FOUND FALSE)
74+
endif ()
75+
76+
mark_as_advanced(
77+
GFLAGS_INCLUDE_DIR
78+
GFLAGS_STATIC_LIB
79+
)

‎cpp/cmake_modules/FindGandiva.cmake‎

Lines changed: 20 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -45,9 +45,7 @@ else()
4545
${GANDIVA_HOME}/include
4646
)
4747

48-
set(GANDIVA_SEARCH_LIB_PATH
49-
${GANDIVA_HOME}/lib
50-
)
48+
set(GANDIVA_SEARCH_LIB_PATH "${GANDIVA_HOME}")
5149

5250
find_path(GANDIVA_INCLUDE_DIR gandiva/expression_registry.h PATHS
5351
${GANDIVA_SEARCH_HEADER_PATHS}
@@ -57,18 +55,30 @@ else()
5755
endif()
5856

5957
find_library(GANDIVA_LIB_PATH NAMES gandiva
60-
PATHS
61-
${GANDIVA_SEARCH_LIB_PATH}
62-
NO_DEFAULT_PATH)
58+
PATHS ${GANDIVA_SEARCH_LIB_PATH}
59+
NO_DEFAULT_PATH
60+
PATH_SUFFIXES "lib")
6361
get_filename_component(GANDIVA_LIBS ${GANDIVA_LIB_PATH} DIRECTORY)
6462

65-
if (GANDIVA_INCLUDE_DIR AND GANDIVA_LIBS)
63+
find_library(GANDIVA_SHARED_LIB_PATH NAMES gandiva
64+
PATHS ${GANDIVA_SEARCH_LIB_PATH}
65+
NO_DEFAULT_PATH
66+
PATH_SUFFIXES "bin" )
67+
68+
get_filename_component(GANDIVA_SHARED_LIBS ${GANDIVA_SHARED_LIB_PATH} PATH )
69+
70+
if (GANDIVA_INCLUDE_DIR AND GANDIVA_LIBS AND GANDIVA_SHARED_LIBS)
6671
set(GANDIVA_FOUND TRUE)
6772
set(GANDIVA_LIB_NAME gandiva)
6873

69-
set(GANDIVA_STATIC_LIB ${GANDIVA_LIBS}/lib${GANDIVA_LIB_NAME}.a)
70-
71-
set(GANDIVA_SHARED_LIB ${GANDIVA_LIBS}/lib${GANDIVA_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
74+
if (MSVC)
75+
set(GANDIVA_STATIC_LIB "${GANDIVA_LIBS}/${GANDIVA_LIB_NAME}${GANDIVA_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}")
76+
set(GANDIVA_SHARED_LIB "${GANDIVA_SHARED_LIBS}/${GANDIVA_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}")
77+
set(GANDIVA_SHARED_IMP_LIB "${GANDIVA_LIBS}/${GANDIVA_LIB_NAME}.lib")
78+
else()
79+
set(GANDIVA_STATIC_LIB ${GANDIVA_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${GANDIVA_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX})
80+
set(GANDIVA_SHARED_LIB ${GANDIVA_SHARED_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${GANDIVA_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
81+
endif()
7282
endif()
7383

7484
if (GANDIVA_FOUND)

‎cpp/cmake_modules/FindParquet.cmake‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,10 @@ if(NOT "$ENV{PARQUET_HOME}" STREQUAL "")
3030
set(PARQUET_HOME "$ENV{PARQUET_HOME}")
3131
endif()
3232

33+
if(NOT "$ENV{ARROW_HOME}" STREQUAL "")
34+
set(ARROW_HOME "$ENV{ARROW_HOME}")
35+
endif()
36+
3337
if (MSVC)
3438
SET(CMAKE_FIND_LIBRARY_SUFFIXES ".lib" ".dll")
3539

@@ -38,7 +42,8 @@ if (MSVC)
3842
endif()
3943

4044
find_library(PARQUET_SHARED_LIBRARIES NAMES parquet
41-
PATHS ${PARQUET_HOME} NO_DEFAULT_PATH
45+
PATHS ${PARQUET_HOME} ${ARROW_HOME}
46+
NO_DEFAULT_PATH
4247
PATH_SUFFIXES "bin" )
4348

4449
get_filename_component(PARQUET_SHARED_LIBS ${PARQUET_SHARED_LIBRARIES} PATH )

‎cpp/cmake_modules/SetupCxxFlags.cmake‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -411,3 +411,22 @@ else()
411411
endif()
412412

413413
message(STATUS "Build Type: ${CMAKE_BUILD_TYPE}")
414+
415+
# ----------------------------------------------------------------------
416+
# MSVC-specific linker options
417+
418+
if (MSVC)
419+
set(MSVC_LINKER_FLAGS)
420+
if (MSVC_LINK_VERBOSE)
421+
set(MSVC_LINKER_FLAGS "${MSVC_LINKER_FLAGS} /VERBOSE:LIB")
422+
endif()
423+
if (NOT ARROW_USE_STATIC_CRT)
424+
set(MSVC_LINKER_FLAGS "${MSVC_LINKER_FLAGS} /NODEFAULTLIB:LIBCMT")
425+
set(CMAKE_EXE_LINKER_FLAGS
426+
"${CMAKE_EXE_LINKER_FLAGS} ${MSVC_LINKER_FLAGS}")
427+
set(CMAKE_MODULE_LINKER_FLAGS
428+
"${CMAKE_MODULE_LINKER_FLAGS} ${MSVC_LINKER_FLAGS}")
429+
set(CMAKE_SHARED_LINKER_FLAGS
430+
"${CMAKE_SHARED_LINKER_FLAGS} ${MSVC_LINKER_FLAGS}")
431+
endif()
432+
endif()

‎cpp/src/gandiva/engine.cc‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,12 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18+
// TODO(wesm): LLVM 7 produces pesky C4244 that disable pragmas around the LLVM
19+
// includes seem to not fix as with LLVM 6
20+
#if defined(_MSC_VER)
21+
#pragma warning(disable : 4244)
22+
#endif
23+
1824
#include "gandiva/engine.h"
1925

2026
#include <iostream>

0 commit comments

Comments
 (0)