ARROW-383: [C++] Integration testing CLI tool

Modeled after Java version in ARROW-367

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #209 from wesm/ARROW-383 and squashes the following commits:

7b29b24 [Wes McKinney] Use git master gflags to avoid valgrind error
f563d3a [Wes McKinney] Build integration test as a normal unit test executable, opt-in to integration testing
bdf1f7a [Wes McKinney] Call the RunCommand method instead
dbc6aab [Wes McKinney] Add unit tests for the integration validator
ca1eade [Wes McKinney] Clean up includes
1752249 [Wes McKinney] Draft integration testing CLI tool, untested
b773d0d [Wes McKinney] Add gflags external project and json-integration-test executable stub

Author: wesm

cpp/CMakeLists.txt

@@ -26,6 +26,7 @@ include(ExternalProject)
 set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support")
 set(THIRDPARTY_DIR "${CMAKE_SOURCE_DIR}/thirdparty")
 
+set(GFLAGS_VERSION "2.1.2")
 set(GTEST_VERSION "1.7.0")
 set(GBENCHMARK_VERSION "1.0.0")
 set(FLATBUFFERS_VERSION "1.3.0")
@@ -506,6 +507,49 @@ if(ARROW_BUILD_TESTS)
   if(GTEST_VENDORED)
     add_dependencies(gtest googletest_ep)
   endif()
+
+  # gflags (formerly Googleflags) command line parsing
+  if("$ENV{GFLAGS_HOME}" STREQUAL "")
+    if(APPLE)
+      set(GFLAGS_CMAKE_CXX_FLAGS "-fPIC -std=c++11 -stdlib=libc++")
+    else()
+      set(GFLAGS_CMAKE_CXX_FLAGS "-fPIC")
+    endif()
+
+    set(GFLAGS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/gflags_ep-prefix/src/gflags_ep")
+    ExternalProject_Add(gflags_ep
+      GIT_REPOSITORY https://github.com/gflags/gflags.git
+      GIT_TAG cce68f0c9c5d054017425e6e6fd54f696d36e8ee
+      # URL "https://github.com/gflags/gflags/archive/v${GFLAGS_VERSION}.tar.gz"
+      BUILD_IN_SOURCE 1
+      CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
+                 -DCMAKE_INSTALL_PREFIX=${GFLAGS_PREFIX}
+                 -DBUILD_SHARED_LIBS=OFF
+                 -DBUILD_STATIC_LIBS=ON
+                 -DBUILD_PACKAGING=OFF
+                 -DBUILD_TESTING=OFF
+                 -BUILD_CONFIG_TESTS=OFF
+                 -DINSTALL_HEADERS=ON
+                 -DCMAKE_CXX_FLAGS=${GFLAGS_CMAKE_CXX_FLAGS})
+
+    set(GFLAGS_HOME "${GFLAGS_PREFIX}")
+    set(GFLAGS_INCLUDE_DIR "${GFLAGS_PREFIX}/include")
+    set(GFLAGS_STATIC_LIB "${GFLAGS_PREFIX}/lib/libgflags.a")
+    set(GFLAGS_VENDORED 1)
+  else()
+    set(GFLAGS_VENDORED 0)
+    find_package(GFlags REQUIRED)
+  endif()
+
+  message(STATUS "GFlags include dir: ${GFLAGS_INCLUDE_DIR}")
+  message(STATUS "GFlags static library: ${GFLAGS_STATIC_LIB}")
+  include_directories(SYSTEM ${GFLAGS_INCLUDE_DIR})
+  ADD_THIRDPARTY_LIB(gflags
+    STATIC_LIB ${GFLAGS_STATIC_LIB})
+
+  if(GFLAGS_VENDORED)
+    add_dependencies(gflags gflags_ep)
+  endif()
 endif()
 
 if(ARROW_BUILD_BENCHMARKS)

cpp/cmake_modules/FindGFlags.cmake

@@ -0,0 +1,60 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# - Find GFLAGS (gflags.h, libgflags.a, libgflags.so, and libgflags.so.0)
+# This module defines
+#  GFLAGS_INCLUDE_DIR, directory containing headers
+#  GFLAGS_SHARED_LIB, path to libgflags shared library
+#  GFLAGS_STATIC_LIB, path to libgflags static library
+#  GFLAGS_FOUND, whether gflags has been found
+
+if( NOT "$ENV{GFLAGS_HOME}" STREQUAL "")
+    file( TO_CMAKE_PATH "$ENV{GFLAGS_HOME}" _native_path )
+    list( APPEND _gflags_roots ${_native_path} )
+elseif ( GFlags_HOME )
+    list( APPEND _gflags_roots ${GFlags_HOME} )
+endif()
+
+if ( _gflags_roots )
+  find_path(GFLAGS_INCLUDE_DIR NAMES gflags/gflags.h
+    PATHS ${_gflags_roots}
+    NO_DEFAULT_PATH
+    PATH_SUFFIXES "include" )
+  find_library(GFLAGS_SHARED_LIB NAMES gflags
+    PATHS ${_gflags_roots}
+    NO_DEFAULT_PATH
+    PATH_SUFFIXES "lib" )
+  find_library(GFLAGS_SHARED_LIB NAMES libgflags.a
+    PATHS ${_gflags_roots}
+    NO_DEFAULT_PATH
+    PATH_SUFFIXES "lib" )
+else()
+  find_path(GFLAGS_INCLUDE_DIR gflags/gflags.h
+    # make sure we don't accidentally pick up a different version
+    NO_CMAKE_SYSTEM_PATH
+    NO_SYSTEM_ENVIRONMENT_PATH)
+  find_library(GFLAGS_SHARED_LIB gflags
+    NO_CMAKE_SYSTEM_PATH
+    NO_SYSTEM_ENVIRONMENT_PATH)
+  find_library(GFLAGS_STATIC_LIB libgflags.a
+    NO_CMAKE_SYSTEM_PATH
+    NO_SYSTEM_ENVIRONMENT_PATH)
+endif()
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(GFLAGS REQUIRED_VARS
+  GFLAGS_SHARED_LIB GFLAGS_STATIC_LIB GFLAGS_INCLUDE_DIR)

cpp/src/arrow/io/file.cc

@@ -186,12 +186,13 @@ static inline Status FileOpenWriteable(const std::string& filename, int* fd) {
   memcpy(wpath.data(), filename.data(), filename.size());
   memcpy(wpath.data() + nwchars, L"\0", sizeof(wchar_t));
 
-  errno_actual = _wsopen_s(
-      fd, wpath.data(), _O_WRONLY | _O_CREAT | _O_BINARY, _SH_DENYNO, _S_IWRITE);
+  errno_actual = _wsopen_s(fd, wpath.data(), _O_WRONLY | _O_CREAT | _O_BINARY | _O_TRUNC,
+      _SH_DENYNO, _S_IWRITE);
   ret = *fd;
 
 #else
-  ret = *fd = open(filename.c_str(), O_WRONLY | O_CREAT | O_BINARY, ARROW_WRITE_SHMODE);
+  ret = *fd =
+      open(filename.c_str(), O_WRONLY | O_CREAT | O_BINARY | O_TRUNC, ARROW_WRITE_SHMODE);
 #endif
   return CheckOpenResult(ret, errno_actual, filename.c_str(), filename.size());
 }

cpp/src/arrow/ipc/CMakeLists.txt

@@ -85,6 +85,33 @@ ADD_ARROW_TEST(ipc-json-test)
 ARROW_TEST_LINK_LIBRARIES(ipc-json-test
   ${ARROW_IPC_TEST_LINK_LIBS})
 
+ADD_ARROW_TEST(json-integration-test)
+
+if (APPLE)
+  target_link_libraries(json-integration-test
+    arrow_static
+    arrow_io
+    arrow_ipc
+    gflags
+    gtest
+    boost_filesystem_static
+    boost_system_static
+    dl)
+  set_target_properties(json-integration-test
+    PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
+else()
+  target_link_libraries(json-integration-test
+    arrow_static
+    arrow_io
+    arrow_ipc
+    gflags
+    gtest
+    pthread
+    boost_filesystem_static
+    boost_system_static
+    dl)
+endif()
+
 # make clean will delete the generated file
 set_source_files_properties(Metadata_generated.h PROPERTIES GENERATED TRUE)
 

cpp/src/arrow/ipc/adapter.h

@@ -16,7 +16,7 @@
 // under the License.
 
 // Public API for writing and accessing (with zero copy, if possible) Arrow
-// data in shared memory
+// IPC binary formatted data (e.g. in shared memory, or from some other IO source)
 
 #ifndef ARROW_IPC_ADAPTER_H
 #define ARROW_IPC_ADAPTER_H