Merge branch 'master' into gandiva-cython

Author: pcmoritz

ci/appveyor-cpp-build.bat

@@ -30,6 +30,7 @@ if "%JOB%" == "Static_Crt_Build" (
   pushd cpp\build-debug
 
   cmake -G "%GENERATOR%" ^
+        -DARROW_VERBOSE_THIRDPARTY_BUILD=OFF ^
         -DARROW_USE_STATIC_CRT=ON ^
         -DARROW_BOOST_USE_SHARED=OFF ^
         -DARROW_BUILD_SHARED=OFF ^
@@ -45,6 +46,7 @@ if "%JOB%" == "Static_Crt_Build" (
   pushd cpp\build-release
 
   cmake -G "%GENERATOR%" ^
+        -DARROW_VERBOSE_THIRDPARTY_BUILD=OFF ^
         -DARROW_USE_STATIC_CRT=ON ^
         -DARROW_BOOST_USE_SHARED=OFF ^
         -DARROW_BUILD_SHARED=OFF ^
@@ -70,6 +72,7 @@ if "%JOB%" == "Build_Debug" (
   pushd cpp\build-debug
 
   cmake -G "%GENERATOR%" ^
+        -DARROW_VERBOSE_THIRDPARTY_BUILD=OFF ^
         -DARROW_BOOST_USE_SHARED=OFF ^
         -DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
         -DARROW_BUILD_STATIC=OFF ^

ci/cpp-msvc-build-main.bat

@@ -19,7 +19,7 @@
 @rem (i.e. for usual configurations)
 
 set ARROW_HOME=%CONDA_PREFIX%\Library
-set CMAKE_ARGS=
+set CMAKE_ARGS=-DARROW_VERBOSE_THIRDPARTY_BUILD=OFF
 
 if "%JOB%" == "Toolchain" (
     set CMAKE_ARGS=%CMAKE_ARGS% -DARROW_WITH_BZ2=ON

ci/travis_script_rust.sh

@@ -35,6 +35,7 @@ cargo rustc -- -D warnings
 
 cargo build
 cargo test
+cargo bench
 cargo run --example dynamic_types
 
 popd

cpp/cmake_modules/ThirdpartyToolchain.cmake

@@ -580,7 +580,7 @@ if(ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS)
                           -DBUILD_STATIC_LIBS=ON
                           -DBUILD_PACKAGING=OFF
                           -DBUILD_TESTING=OFF
-                          -BUILD_CONFIG_TESTS=OFF
+                          -DBUILD_CONFIG_TESTS=OFF
                           -DINSTALL_HEADERS=ON
                           -DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_CXX_FLAGS}
                           -DCMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_C_FLAGS}
@@ -1053,38 +1053,38 @@ if (ARROW_WITH_ZSTD)
 # ZSTD
 
   if("${ZSTD_HOME}" STREQUAL "")
-    set(ZSTD_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/zstd_ep-prefix/src/zstd_ep")
-    set(ZSTD_INCLUDE_DIR "${ZSTD_BUILD_DIR}/lib")
+    set(ZSTD_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/zstd_ep-install")
+    set(ZSTD_INCLUDE_DIR "${ZSTD_PREFIX}/include")
+
+    set(ZSTD_CMAKE_ARGS
+        "-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}"
+        "-DCMAKE_INSTALL_PREFIX=${ZSTD_PREFIX}"
+        "-DZSTD_BUILD_PROGRAMS=off"
+        "-DZSTD_BUILD_SHARED=off"
+        "-DZSTD_BUILD_STATIC=on"
+        "-DZSTD_MULTITHREAD_SUPPORT=off")
 
     if (MSVC)
+      set(ZSTD_STATIC_LIB "${ZSTD_PREFIX}/lib/zstd_static.lib")
       if (ARROW_USE_STATIC_CRT)
-        if (${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG")
-          set(ZSTD_RUNTIME_LIBRARY_LINKAGE "/p:RuntimeLibrary=MultiThreadedDebug")
-        else()
-          set(ZSTD_RUNTIME_LIBRARY_LINKAGE "/p:RuntimeLibrary=MultiThreaded")
-        endif()
+        set(ZSTD_CMAKE_ARGS ${ZSTD_CMAKE_ARGS} "-DZSTD_USE_STATIC_RUNTIME=on")
       endif()
-      set(ZSTD_STATIC_LIB "${ZSTD_BUILD_DIR}/build/VS2010/bin/x64_${CMAKE_BUILD_TYPE}/libzstd_static.lib")
-      set(ZSTD_BUILD_COMMAND BUILD_COMMAND msbuild ${ZSTD_BUILD_DIR}/build/VS2010/zstd.sln /t:Build /v:minimal /p:Configuration=${CMAKE_BUILD_TYPE}
-                             ${ZSTD_RUNTIME_LIBRARY_LINKAGE} /p:Platform=x64 /p:PlatformToolset=v140
-                             /p:OutDir=${ZSTD_BUILD_DIR}/build/VS2010/bin/x64_${CMAKE_BUILD_TYPE}/ /p:SolutionDir=${ZSTD_BUILD_DIR}/build/VS2010/ )
-      set(ZSTD_PATCH_COMMAND PATCH_COMMAND git --git-dir=. apply --verbose --whitespace=fix ${CMAKE_SOURCE_DIR}/build-support/zstd_msbuild_gl_runtimelibrary_params.patch)
     else()
-      set(ZSTD_STATIC_LIB "${ZSTD_BUILD_DIR}/lib/libzstd.a")
-      set(ZSTD_BUILD_COMMAND BUILD_COMMAND ${CMAKE_SOURCE_DIR}/build-support/build-zstd-lib.sh)
+      set(ZSTD_STATIC_LIB "${ZSTD_PREFIX}/lib/libzstd.a")
+      # Only pass our C flags on Unix as on MSVC it leads to a
+      # "incompatible command-line options" error
+      set(ZSTD_CMAKE_ARGS ${ZSTD_CMAKE_ARGS}
+          "-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}"
+          "-DCMAKE_C_FLAGS=${EP_C_FLAGS}")
     endif()
 
     ExternalProject_Add(zstd_ep
-        URL ${ZSTD_SOURCE_URL}
-        ${EP_LOG_OPTIONS}
-        UPDATE_COMMAND ""
-        ${ZSTD_PATCH_COMMAND}
-        CONFIGURE_COMMAND ""
-        INSTALL_COMMAND ""
-        BINARY_DIR ${ZSTD_BUILD_DIR}
-        BUILD_BYPRODUCTS ${ZSTD_STATIC_LIB}
-        ${ZSTD_BUILD_COMMAND}
-        )
+      ${EP_LOG_OPTIONS}
+      CMAKE_ARGS ${ZSTD_CMAKE_ARGS}
+      SOURCE_SUBDIR "build/cmake"
+      INSTALL_DIR ${ZSTD_PREFIX}
+      URL ${ZSTD_SOURCE_URL}
+      BUILD_BYPRODUCTS "${ZSTD_STATIC_LIB}")
 
     set(ZSTD_VENDORED 1)
   else()

cpp/src/arrow/buffer-test.cc

@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#include <algorithm>
 #include <cstdint>
 #include <cstring>
 #include <limits>
@@ -272,7 +273,10 @@ TYPED_TEST(TypedTestBuffer, ResizeOOM) {
   TypeParam buf;
   ASSERT_OK(AllocateResizableBuffer(0, &buf));
   ASSERT_OK(buf->Resize(100));
-  int64_t to_alloc = std::numeric_limits<int64_t>::max();
+  int64_t to_alloc = std::min<uint64_t>(std::numeric_limits<int64_t>::max(),
+                                        std::numeric_limits<size_t>::max());
+  // subtract 63 to prevent overflow after the size is aligned
+  to_alloc -= 63;
   ASSERT_RAISES(OutOfMemory, buf->Resize(to_alloc));
 #endif
 }

cpp/src/arrow/builder.cc

@@ -736,7 +736,7 @@ Status BooleanBuilder::AppendValues(const std::vector<bool>& values,
 
   int64_t i = 0;
   internal::GenerateBitsUnrolled(raw_data_, length_, length,
-                                 [values, &i]() -> bool { return values[i++]; });
+                                 [&values, &i]() -> bool { return values[i++]; });
 
   // this updates length_
   ArrayBuilder::UnsafeAppendToBitmap(is_valid);
@@ -749,7 +749,7 @@ Status BooleanBuilder::AppendValues(const std::vector<bool>& values) {
 
   int64_t i = 0;
   internal::GenerateBitsUnrolled(raw_data_, length_, length,
-                                 [values, &i]() -> bool { return values[i++]; });
+                                 [&values, &i]() -> bool { return values[i++]; });
 
   // this updates length_
   ArrayBuilder::UnsafeSetNotNull(length);

cpp/src/arrow/memory_pool-test.h

@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#include <algorithm>
 #include <cstdint>
 #include <limits>
 
@@ -53,7 +54,10 @@ class TestMemoryPoolBase : public ::testing::Test {
     auto pool = memory_pool();
 
     uint8_t* data;
-    int64_t to_alloc = std::numeric_limits<int64_t>::max();
+    int64_t to_alloc = std::min<uint64_t>(std::numeric_limits<int64_t>::max(),
+                                          std::numeric_limits<size_t>::max());
+    // subtract 63 to prevent overflow after the size is aligned
+    to_alloc -= 63;
     ASSERT_RAISES(OutOfMemory, pool->Allocate(to_alloc, &data));
   }
 

cpp/src/arrow/memory_pool.cc

@@ -23,6 +23,7 @@
 #include <cstdlib>
 #include <cstring>
 #include <iostream>
+#include <limits>
 #include <memory>
 #include <sstream>  // IWYU pragma: keep
 
@@ -44,7 +45,13 @@ namespace {
 // Allocate memory according to the alignment requirements for Arrow
 // (as of May 2016 64 bytes)
 Status AllocateAligned(int64_t size, uint8_t** out) {
-// TODO(emkornfield) find something compatible with windows
+  // TODO(emkornfield) find something compatible with windows
+  if (size < 0) {
+    return Status::Invalid("negative malloc size");
+  }
+  if (static_cast<uint64_t>(size) >= std::numeric_limits<size_t>::max()) {
+    return Status::CapacityError("malloc size overflows size_t");
+  }
 #ifdef _WIN32
   // Special code path for Windows
   *out =
@@ -104,7 +111,14 @@ class DefaultMemoryPool : public MemoryPool {
   Status Reallocate(int64_t old_size, int64_t new_size, uint8_t** ptr) override {
 #ifdef ARROW_JEMALLOC
     uint8_t* previous_ptr = *ptr;
-    *ptr = reinterpret_cast<uint8_t*>(rallocx(*ptr, new_size, MALLOCX_ALIGN(kAlignment)));
+    if (new_size < 0) {
+      return Status::Invalid("negative realloc size");
+    }
+    if (static_cast<uint64_t>(new_size) >= std::numeric_limits<size_t>::max()) {
+      return Status::CapacityError("realloc overflows size_t");
+    }
+    *ptr = reinterpret_cast<uint8_t*>(
+        rallocx(*ptr, static_cast<size_t>(new_size), MALLOCX_ALIGN(kAlignment)));
     if (*ptr == NULL) {
       std::stringstream ss;
       ss << "realloc of size " << new_size << " failed";

cpp/src/arrow/util/bit-util.h

@@ -170,7 +170,7 @@ static inline int CountLeadingZeros(uint32_t value) {
 static inline int CountLeadingZeros(uint64_t value) {
 #if defined(__clang__) || defined(__GNUC__)
   if (value == 0) return 64;
-  return static_cast<int>(__builtin_clzl(value));
+  return static_cast<int>(__builtin_clzll(value));
 #elif defined(_MSC_VER)
   unsigned long index;                     // NOLINT
   if (_BitScanReverse64(&index, value)) {  // NOLINT

cpp/src/arrow/util/compression_zstd.cc

@@ -24,6 +24,7 @@
 #include <zstd.h>
 
 #include "arrow/status.h"
+#include "arrow/util/logging.h"
 #include "arrow/util/macros.h"
 
 using std::size_t;
@@ -34,6 +35,12 @@ namespace util {
 // XXX level = 1 probably doesn't compress very much
 constexpr int kZSTDDefaultCompressionLevel = 1;
 
+static Status ZSTDError(size_t ret, const char* prefix_msg) {
+  std::stringstream ss;
+  ss << prefix_msg << ZSTD_getErrorName(ret);
+  return Status::IOError(ss.str());
+}
+
 // ----------------------------------------------------------------------
 // ZSTD decompressor implementation
 
@@ -47,7 +54,7 @@ class ZSTDDecompressor : public Decompressor {
     finished_ = false;
     size_t ret = ZSTD_initDStream(stream_);
     if (ZSTD_isError(ret)) {
-      return ZSTDError(ret, "zstd init failed: ");
+      return ZSTDError(ret, "ZSTD init failed: ");
     } else {
       return Status::OK();
     }
@@ -69,7 +76,7 @@ class ZSTDDecompressor : public Decompressor {
     size_t ret;
     ret = ZSTD_decompressStream(stream_, &out_buf, &in_buf);
     if (ZSTD_isError(ret)) {
-      return ZSTDError(ret, "zstd decompress failed: ");
+      return ZSTDError(ret, "ZSTD decompress failed: ");
     }
     *bytes_read = static_cast<int64_t>(in_buf.pos);
     *bytes_written = static_cast<int64_t>(out_buf.pos);
@@ -81,12 +88,6 @@ class ZSTDDecompressor : public Decompressor {
   bool IsFinished() override { return finished_; }
 
  protected:
-  Status ZSTDError(size_t ret, const char* prefix_msg) {
-    std::stringstream ss;
-    ss << prefix_msg << ZSTD_getErrorName(ret);
-    return Status::IOError(ss.str());
-  }
-
   ZSTD_DStream* stream_;
   bool finished_;
 };
@@ -103,7 +104,7 @@ class ZSTDCompressor : public Compressor {
   Status Init() {
     size_t ret = ZSTD_initCStream(stream_, kZSTDDefaultCompressionLevel);
     if (ZSTD_isError(ret)) {
-      return ZSTDError(ret, "zstd init failed: ");
+      return ZSTDError(ret, "ZSTD init failed: ");
     } else {
       return Status::OK();
     }
@@ -119,12 +120,6 @@ class ZSTDCompressor : public Compressor {
              bool* should_retry) override;
 
  protected:
-  Status ZSTDError(size_t ret, const char* prefix_msg) {
-    std::stringstream ss;
-    ss << prefix_msg << ZSTD_getErrorName(ret);
-    return Status::IOError(ss.str());
-  }
-
   ZSTD_CStream* stream_;
 };
 
@@ -144,7 +139,7 @@ Status ZSTDCompressor::Compress(int64_t input_len, const uint8_t* input,
   size_t ret;
   ret = ZSTD_compressStream(stream_, &out_buf, &in_buf);
   if (ZSTD_isError(ret)) {
-    return ZSTDError(ret, "zstd compress failed: ");
+    return ZSTDError(ret, "ZSTD compress failed: ");
   }
   *bytes_read = static_cast<int64_t>(in_buf.pos);
   *bytes_written = static_cast<int64_t>(out_buf.pos);
@@ -162,7 +157,7 @@ Status ZSTDCompressor::Flush(int64_t output_len, uint8_t* output, int64_t* bytes
   size_t ret;
   ret = ZSTD_flushStream(stream_, &out_buf);
   if (ZSTD_isError(ret)) {
-    return ZSTDError(ret, "zstd flush failed: ");
+    return ZSTDError(ret, "ZSTD flush failed: ");
   }
   *bytes_written = static_cast<int64_t>(out_buf.pos);
   *should_retry = ret > 0;
@@ -180,7 +175,7 @@ Status ZSTDCompressor::End(int64_t output_len, uint8_t* output, int64_t* bytes_w
   size_t ret;
   ret = ZSTD_endStream(stream_, &out_buf);
   if (ZSTD_isError(ret)) {
-    return ZSTDError(ret, "zstd end failed: ");
+    return ZSTDError(ret, "ZSTD end failed: ");
   }
   *bytes_written = static_cast<int64_t>(out_buf.pos);
   *should_retry = ret > 0;
@@ -206,10 +201,20 @@ Status ZSTDCodec::MakeDecompressor(std::shared_ptr<Decompressor>* out) {
 
 Status ZSTDCodec::Decompress(int64_t input_len, const uint8_t* input, int64_t output_len,
                              uint8_t* output_buffer) {
-  int64_t decompressed_size =
-      ZSTD_decompress(output_buffer, static_cast<size_t>(output_len), input,
-                      static_cast<size_t>(input_len));
-  if (decompressed_size != output_len) {
+  if (output_buffer == nullptr) {
+    // We may pass a NULL 0-byte output buffer but some zstd versions demand
+    // a valid pointer: https://github.com/facebook/zstd/issues/1385
+    static uint8_t empty_buffer[1];
+    DCHECK_EQ(output_len, 0);
+    output_buffer = empty_buffer;
+  }
+
+  size_t ret = ZSTD_decompress(output_buffer, static_cast<size_t>(output_len), input,
+                               static_cast<size_t>(input_len));
+  if (ZSTD_isError(ret)) {
+    return ZSTDError(ret, "ZSTD decompression failed: ");
+  }
+  if (static_cast<int64_t>(ret) != output_len) {
     return Status::IOError("Corrupt ZSTD compressed data.");
   }
   return Status::OK();
@@ -223,12 +228,13 @@ int64_t ZSTDCodec::MaxCompressedLen(int64_t input_len,
 Status ZSTDCodec::Compress(int64_t input_len, const uint8_t* input,
                            int64_t output_buffer_len, uint8_t* output_buffer,
                            int64_t* output_length) {
-  *output_length =
+  size_t ret =
       ZSTD_compress(output_buffer, static_cast<size_t>(output_buffer_len), input,
                     static_cast<size_t>(input_len), kZSTDDefaultCompressionLevel);
-  if (ZSTD_isError(*output_length)) {
-    return Status::IOError("ZSTD compression failure.");
+  if (ZSTD_isError(ret)) {
+    return ZSTDError(ret, "ZSTD compression failed: ");
   }
+  *output_length = static_cast<int64_t>(ret);
   return Status::OK();
 }