Merge branch 'branch-0.6' into fea-ext-string-support

CHANGELOG.md

@@ -105,6 +105,7 @@
 - PR #1108 Sorting for datetime columns
 - PR #1120 Return a `Series` (not a `Column`) from `Series.cat.set_categories()`
 - PR #1128 CSV Reader: The last data row does not need to be line terminated
+- PR #1183 Bump Arrow version to 0.12.1
 
 ## Bug Fixes
 
@@ -146,11 +147,14 @@
 - PR #1090 Updating Doxygen Comments
 - PR #1080 Fix dtypes returned from loc / iloc because of lists
 - PR #1102 CSV Reader: Minor fixes and memory usage improvements
+- PR #1174: Fix release script typo
 - PR #1137 Add prebuild script for CI
 - PR #1118 Enhanced the `DataFrame.from_records()` feature
 - PR #1129 Fix join performance with index parameter from using numpy array
 - PR #1145 Issue with .agg call on multi-column dataframes
+- PR #908 Some testing code cleanup
 - PR #1167 Fix issue with null_count not being set after inplace fillna()
+- PR #1185 Support left_on/right_on and also on=str in merge
 
 
 # cuDF 0.5.1 (05 Feb 2019)

ci/release/update-version.sh

@@ -32,7 +32,7 @@ elif [ "$RELEASE_TYPE" == "minor" ]; then
   NEXT_SHORT_TAG="${CURRENT_MAJOR}.${NEXT_MINOR}"
 elif [ "$RELEASE_TYPE" == "patch" ]; then
   NEXT_FULL_TAG="${CURRENT_MAJOR}.${CURRENT_MINOR}.${NEXT_PATCH}"
-  NEXT_SHORT_TAG="${CURRENT_MAJOR}.${NEXT_MINOR}"
+  NEXT_SHORT_TAG="${CURRENT_MAJOR}.${CURRENT_MINOR}"
 else
   echo "Incorrect release type; use 'major', 'minor', or 'patch' as an argument"
   exit 1

conda/environments/cudf_dev.yml

@@ -10,7 +10,7 @@ dependencies:
 - python>=3.6,<3.8
 - numba>=0.40
 - pandas>=0.23.4
-- pyarrow=0.12.0
+- pyarrow=0.12.1
 - notebook>=0.5.0
 - boost
 - nvstrings

conda/recipes/cudf/meta.yaml

@@ -45,7 +45,7 @@ requirements:
 
 test:
   requires:
-    - distributed>=1.22.1
+    - distributed>=1.23.0
     - pytest
   imports:
     - cudf

conda/recipes/libcudf/meta.yaml

@@ -34,7 +34,7 @@ requirements:
     - cmake>=3.12.4
     - nvstrings=0.3.0dev0
   run:
-    - pyarrow=0.12.0
+    - pyarrow=0.12.1
     - nvstrings=0.3.0dev0
 
 test:

conda/recipes/libcudf_cffi/meta.yaml

@@ -38,13 +38,13 @@ requirements:
     - setuptools
     - nvstrings=0.3.0dev0
     - pycparser=2.19
-    - pyarrow=0.12.0
+    - pyarrow=0.12.1
   run:
     - libcudf={{ version }}
     - python x.x
     - cffi
     - pycparser=2.19
-    - pyarrow=0.12.0
+    - pyarrow=0.12.1
 
 test:
   commands:

cpp/cmake/Modules/ConfigureArrow.cmake

@@ -97,7 +97,7 @@ set(FLATBUFFERS_INCLUDE_DIR "${FLATBUFFERS_ROOT}/include")
 set(FLATBUFFERS_LIBRARY_DIR "${FLATBUFFERS_ROOT}/lib")
 
 add_definitions(-DARROW_METADATA_V4)
-add_definitions(-DARROW_VERSION=1200)
+add_definitions(-DARROW_VERSION=1210)
 
 
 

cpp/cmake/Templates/Arrow.CMakeLists.txt.cmake

@@ -4,7 +4,7 @@ include(ExternalProject)
 
 ExternalProject_Add(Arrow
                     GIT_REPOSITORY    https://github.com/apache/arrow.git
-                    GIT_TAG           apache-arrow-0.12.0
+                    GIT_TAG           apache-arrow-0.12.1
                     SOURCE_DIR        "${ARROW_ROOT}/arrow"
                     SOURCE_SUBDIR     "cpp"
                     BINARY_DIR        "${ARROW_ROOT}/build"

cpp/include/cudf/types.h

@@ -104,10 +104,10 @@ typedef struct {
  */
 // TODO: #1119 Use traits to set `gdf_data` elements
 typedef union {
-  int8_t        si08;  /**< GDF_INT8      */
-  int16_t       si16;  /**< GDF_INT16     */
-  int32_t       si32;  /**< GDF_INT32     */
-  int64_t       si64;  /**< GDF_INT64     */
+  char          si08;  /**< GDF_INT8      */
+  short         si16;  /**< GDF_INT16     */
+  int           si32;  /**< GDF_INT32     */
+  long          si64;  /**< GDF_INT64     */
   float         fp32;  /**< GDF_FLOAT32   */
   double        fp64;  /**< GDF_FLOAT64   */
   gdf_date32    dt32;  /**< GDF_DATE32    */

cpp/src/utilities/bit_util.cuh

@@ -17,6 +17,23 @@
  */
 #pragma once
 
+#include <cudf/types.h>
+
+#include <stdint.h>
+#include <string>
+
+#ifndef CUDA_HOST_DEVICE_CALLABLE
+#ifdef __CUDACC__
+#define CUDA_HOST_DEVICE_CALLABLE __host__ __device__ inline
+#define CUDA_DEVICE_CALLABLE __device__ inline
+#define CUDA_LAUNCHABLE __global__
+#else
+#define CUDA_HOST_DEVICE_CALLABLE inline
+#define CUDA_DEVICE_CALLABLE inline
+#define CUDA_LAUNCHABLE
+#endif
+#endif
+
 namespace gdf {
 namespace util {
 
@@ -25,7 +42,7 @@ using ValidType = uint32_t;
 
 
 // Instead of this function, use gdf_valid_allocation_size from legacy_bitmask.hpp
-//__host__ __device__ __forceinline__
+//CUDA_HOST_DEVICE_CALLABLE
 //  size_t
 //  valid_size(size_t column_length)
 //{
@@ -34,38 +51,38 @@ using ValidType = uint32_t;
 //}
 
 // Instead of this function, use gdf_is_valid from gdf/utils.h
-///__host__ __device__ __forceinline__ bool get_bit(const gdf_valid_type* const bits, size_t i)
+///CUDA_HOST_DEVICE_CALLABLE bool get_bit(const gdf_valid_type* const bits, size_t i)
 ///{
 ///  return  bits == nullptr? true :  bits[i >> size_t(3)] & (1 << (i & size_t(7)));
 ///}
 
-__host__ __device__ __forceinline__
+CUDA_HOST_DEVICE_CALLABLE
   uint8_t
   byte_bitmask(size_t i)
 {
   static uint8_t kBitmask[8] = { 1, 2, 4, 8, 16, 32, 64, 128 };
   return kBitmask[i];
 }
 
-__host__ __device__ __forceinline__
+CUDA_HOST_DEVICE_CALLABLE
   uint8_t
   flipped_bitmask(size_t i)
 {
   static uint8_t kFlippedBitmask[] = { 254, 253, 251, 247, 239, 223, 191, 127 };
   return kFlippedBitmask[i];
 }
 
-__host__ __device__ __forceinline__ void turn_bit_on(uint8_t* const bits, size_t i)
+CUDA_HOST_DEVICE_CALLABLE void turn_bit_on(uint8_t* const bits, size_t i)
 {
   bits[i / 8] |= byte_bitmask(i % 8);
 }
 
-__host__ __device__ __forceinline__ void turn_bit_off(uint8_t* const bits, size_t i)
+CUDA_HOST_DEVICE_CALLABLE void turn_bit_off(uint8_t* const bits, size_t i)
 {
   bits[i / 8] &= flipped_bitmask(i % 8);
 }
 
-__host__ __device__ __forceinline__ size_t last_byte_index(size_t column_size)
+CUDA_HOST_DEVICE_CALLABLE size_t last_byte_index(size_t column_size)
 {
   return (column_size + 8 - 1) / 8;
 }

cpp/src/utilities/cudf_utils.h

@@ -1,10 +1,12 @@
 #ifndef GDF_UTILS_H
 #define GDF_UTILS_H
 
+#include <utilities/error_utils.hpp>
+#include <cudf.h>
+
 #include <cuda_runtime_api.h>
+
 #include <vector>
-#include "cudf.h"
-#include "error_utils.hpp"
 
 #ifdef __CUDACC__
 #define CUDA_HOST_DEVICE_CALLABLE __host__ __device__ inline

cpp/src/utilities/type_dispatcher.hpp

@@ -20,6 +20,11 @@
 #include "cudf/types.h"
 #include "wrapper_types.hpp"
 #include "release_assert.cuh"
+
+#include <cudf/types.h>
+
+#include <NVStrings.h>
+
 #include <cassert>
 #include <utility>
 

cpp/src/utilities/wrapper_types.hpp

@@ -1,9 +1,10 @@
 #ifndef GDF_CPPTYPES_H
 #define GDF_CPPTYPES_H
 
-#include "cudf/types.h"
+#include <cudf/types.h>
 #include "cudf_utils.h"
-#include <iostream>
+
+#include <iosfwd>
 #include <type_traits>
 
 /* --------------------------------------------------------------------------*/

cpp/tests/CMakeLists.txt

@@ -64,6 +64,7 @@ ConfigureTest(ERROR_TEST "${ERROR_TEST_SRC}")
 # - filter tests ----------------------------------------------------------------------------------
 
 set(FILTER_TEST_SRC
+    "${CMAKE_CURRENT_SOURCE_DIR}/utilities/cudf_test_utils.cu"
     "${CMAKE_CURRENT_SOURCE_DIR}/filter/helper/utils.cuh"
     "${CMAKE_CURRENT_SOURCE_DIR}/filter/helper/utils.cu"
     "${CMAKE_CURRENT_SOURCE_DIR}/filter/test_example.cu"
@@ -83,6 +84,7 @@ ConfigureTest(GROUPBY_TEST "${GROUPBY_TEST_SRC}")
 # - join tests ------------------------------------------------------------------------------------
 
 set(JOIN_TEST_SRC 
+    "${CMAKE_CURRENT_SOURCE_DIR}/utilities/valid_vectors.cpp"
     "${CMAKE_CURRENT_SOURCE_DIR}/join/join_tests.cu")
 
 ConfigureTest(JOIN_TEST "${JOIN_TEST_SRC}")
@@ -91,6 +93,7 @@ ConfigureTest(JOIN_TEST "${JOIN_TEST_SRC}")
 # - orderby tests ---------------------------------------------------------------------------------
 
 set(ORDERBY_TEST_SRC 
+    "${CMAKE_CURRENT_SOURCE_DIR}/utilities/valid_vectors.cpp"
     "${CMAKE_CURRENT_SOURCE_DIR}/orderby/orderby_tests.cu")
 
 ConfigureTest(ORDERBY_TEST "${ORDERBY_TEST_SRC}")

cpp/tests/bitmask/bit_mask_test.cu

@@ -14,17 +14,20 @@
  * limitations under the License.
  */
 
-#include "gtest/gtest.h"
-#include "gmock/gmock.h"
+#include <tests/utilities/cudf_test_utils.cuh>
+#include <tests/utilities/cudf_test_fixtures.h>
+#include <bitmask/bit_mask.cuh>
 
-#include "cuda_profiler_api.h"
+#include <gtest/gtest.h>
+#include <gmock/gmock.h>
 
-#include "tests/utilities/cudf_test_utils.cuh"
-#include "tests/utilities/cudf_test_fixtures.h"
-#include "bitmask/bit_mask.cuh"
+#include <cuda_profiler_api.h>
+
+#include <gtest/gtest.h>
 
 #include <chrono>
 
+
 struct BitMaskTest : public GdfTest {};
 
 //

cpp/tests/bitmask/valid_ops_test.cu

@@ -14,16 +14,15 @@
  * limitations under the License.
  */
 
-#include "gtest/gtest.h"
-#include "gmock/gmock.h"
+#include <tests/utilities/cudf_test_utils.cuh>
+#include <tests/utilities/cudf_test_fixtures.h>
 
 #include <cudf.h>
-#include <cudf/functions.h>
 
-#include "cuda_profiler_api.h"
+#include <gtest/gtest.h>
+#include <gmock/gmock.h>
 
-#include "tests/utilities/cudf_test_utils.cuh"
-#include "tests/utilities/cudf_test_fixtures.h"
+#include <cuda_profiler_api.h>
 
 #include <chrono>
 

cpp/tests/column/column_test.cu

@@ -1,20 +1,20 @@
-#include "gtest/gtest.h"
+#include <tests/utilities/cudf_test_utils.cuh>
+
+#include <utilities/cudf_utils.h>
+#include <cudf.h>
+
+#include <thrust/device_vector.h>
+
+#include <gtest/gtest.h>
+
+#include <gtest/gtest.h>
 
 #include <cstdlib>
 #include <iostream>
 #include <vector>
 #include <chrono>
 #include <map>
 
-#include <thrust/device_vector.h>
-
-#include "gtest/gtest.h"
-
-#include <cudf.h>
-#include <utilities/cudf_utils.h>
-#include <cudf/functions.h>
-
-#include "tests/utilities/cudf_test_utils.cuh"
 
 // uncomment to enable benchmarking gdf_column_concat
 //#define ENABLE_CONCAT_BENCHMARK 

cpp/tests/datetime/datetime_ops_test.cu

@@ -16,16 +16,19 @@
  * limitations under the License.
  */
 
-#include <cstdlib>
-#include <iostream>
-#include <vector>
-#include <thrust/device_vector.h>
+#include <tests/utilities/cudf_test_fixtures.h>
 
 #include <cudf.h>
 #include <cudf/functions.h>
+
 #include <rmm/thrust_rmm_allocator.h>
 
-#include "tests/utilities/cudf_test_fixtures.h"
+#include <thrust/device_vector.h>
+
+#include <iostream>
+#include <vector>
+
+#include <cstdlib>
 
 
 struct gdf_extract_from_datetime_example_test : public GdfTest {};

cpp/tests/error/error_handling_test.cu

@@ -13,12 +13,13 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-#include "cudf.h"
-#include "gtest/gtest.h"
-#include "utilities/error_utils.hpp"
+#include <utilities/error_utils.hpp>
+#include <cudf.h>
 
-#include <cuda_runtime_api.h>
 #include <rmm/rmm.h>
+
+#include <gtest/gtest.h>
+
 #include <cstring>
 
 // If this test fails, it means an error code was added without

cpp/tests/filter/helper/utils.cu

@@ -1,12 +1,8 @@
 
-#include <iostream>
-#include <cudf.h>
-#include <cudf/functions.h>
-#include <cuda_runtime.h>
-#include <limits.h>
-#include <gtest/gtest.h>
 #include "utils.cuh"
 
+#include <climits>
+
 
 gdf_valid_type * get_gdf_valid_from_device(gdf_column* column) {
     gdf_valid_type * host_valid_out;