ARROW-373: [C++] JSON serialization format for testing

C++ version of ARROW-372

Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #202 from wesm/ARROW-373 and squashes the following commits:

d13a05f [Wes McKinney] Compiler warning
72c24fe [Wes McKinney] Add a minimal literal JSON example
a2cf47b [Wes McKinney] cpplint
3d9fcc2 [Wes McKinney] Complete round trip json file test with multiple record batches
2753449 [Wes McKinney] Complete draft json roundtrip implementation. tests not complete yet
3d6bbbd [Wes McKinney] Start high level writer scaffold
6bbd669 [Wes McKinney] Tweaks
e2e86b5 [Wes McKinney] Test JSON array roundtrip for numeric types, strings, lists, structs
82f108b [Wes McKinney] Refactoring. Array test scaffold
0891378 [Wes McKinney] Declare loop variables
6566343 [Wes McKinney] Recursively construct children for list/struct
35c2f85 [Wes McKinney] Refactoring. Start drafting string/list reader
f26402a [Wes McKinney] Install type_traits.h. cpplint
4fc7294 [Wes McKinney] Refactoring, type attribute consistency. Array reader compiles
2c93cce [Wes McKinney] WIP JSON array reader code path
932ba7a [Wes McKinney] Add ArrayVisitor methods, add enough metaprogramming to detect presence of c_type type member
15c1094 [Wes McKinney] Add type traits, refactoring, drafting json array writing. not working yet
209ba48 [Wes McKinney] More types refactoring. Strange linker error in pyarrow
379da3c [Wes McKinney] Implement union metadata JSON serialization
5fbea41 [Wes McKinney] Implement some more json types and add convenience factory functions
1c08233 [Wes McKinney] JSON schema roundtrip passing for many types
86c9559 [Wes McKinney] Add convenience factory functions for common types
3b9d14e [Wes McKinney] Add type-specific JSON metadata to schema writer
820b0f2 [Wes McKinney] Drafting JSON schema read/write
68ee7ab [Wes McKinney] Move forward declarations into type_fwd.h
1edf2a9 [Wes McKinney] Prototyping out visitor pattern for json serialization
24c1d5d [Wes McKinney] Some Types refactoring, add TypeVisitor abstract class. Add RapidJSON as external project

Author: wesm

cpp/CMakeLists.txt

@@ -545,6 +545,25 @@ if(ARROW_BUILD_BENCHMARKS)
   endif()
 endif()
 
+# RapidJSON, header only dependency
+if("$ENV{RAPIDJSON_HOME}" STREQUAL "")
+  ExternalProject_Add(rapidjson_ep
+    PREFIX "${CMAKE_BINARY_DIR}"
+    URL "https://github.com/miloyip/rapidjson/archive/v1.1.0.tar.gz"
+    URL_MD5 "badd12c511e081fec6c89c43a7027bce"
+    CONFIGURE_COMMAND ""
+    BUILD_COMMAND ""
+    BUILD_IN_SOURCE 1
+    INSTALL_COMMAND "")
+
+  ExternalProject_Get_Property(rapidjson_ep SOURCE_DIR)
+  set(RAPIDJSON_INCLUDE_DIR "${SOURCE_DIR}/include")
+else()
+  set(RAPIDJSON_INCLUDE_DIR "$ENV{RAPIDJSON_HOME}/include")
+endif()
+message(STATUS "RapidJSON include dir: ${RAPIDJSON_INCLUDE_DIR}")
+include_directories(SYSTEM ${RAPIDJSON_INCLUDE_DIR})
+
 ## Google PerfTools
 ##
 ## Disabled with TSAN/ASAN as well as with gold+dynamic linking (see comment

cpp/src/arrow/CMakeLists.txt

@@ -24,6 +24,8 @@ install(FILES
   schema.h
   table.h
   type.h
+  type_fwd.h
+  type_traits.h
   test-util.h
   DESTINATION include/arrow)
 

cpp/src/arrow/array.cc

@@ -18,13 +18,24 @@
 #include "arrow/array.h"
 
 #include <cstdint>
+#include <cstring>
 
 #include "arrow/util/bit-util.h"
 #include "arrow/util/buffer.h"
 #include "arrow/util/status.h"
 
 namespace arrow {
 
+Status GetEmptyBitmap(
+    MemoryPool* pool, int32_t length, std::shared_ptr<MutableBuffer>* result) {
+  auto buffer = std::make_shared<PoolBuffer>(pool);
+  RETURN_NOT_OK(buffer->Resize(BitUtil::BytesForBits(length)));
+  memset(buffer->mutable_data(), 0, buffer->size());
+
+  *result = buffer;
+  return Status::OK();
+}
+
 // ----------------------------------------------------------------------
 // Base array class
 
@@ -66,4 +77,8 @@ bool NullArray::RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_st
   return true;
 }
 
+Status NullArray::Accept(ArrayVisitor* visitor) const {
+  return visitor->Visit(*this);
+}
+
 }  // namespace arrow

cpp/src/arrow/array.h

@@ -29,6 +29,8 @@
 namespace arrow {
 
 class Buffer;
+class MemoryPool;
+class MutableBuffer;
 class Status;
 
 // Immutable data array with some logical type and some length. Any memory is
@@ -70,6 +72,8 @@ class ARROW_EXPORT Array {
   // returning Status::OK.  This can be an expensive check.
   virtual Status Validate() const;
 
+  virtual Status Accept(ArrayVisitor* visitor) const = 0;
+
  protected:
   std::shared_ptr<DataType> type_;
   int32_t null_count_;
@@ -86,6 +90,8 @@ class ARROW_EXPORT Array {
 // Degenerate null type Array
 class ARROW_EXPORT NullArray : public Array {
  public:
+  using TypeClass = NullType;
+
   NullArray(const std::shared_ptr<DataType>& type, int32_t length)
       : Array(type, length, length, nullptr) {}
 
@@ -94,9 +100,15 @@ class ARROW_EXPORT NullArray : public Array {
   bool Equals(const std::shared_ptr<Array>& arr) const override;
   bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_index,
       const std::shared_ptr<Array>& arr) const override;
+
+  Status Accept(ArrayVisitor* visitor) const override;
 };
 
 typedef std::shared_ptr<Array> ArrayPtr;
+
+Status ARROW_EXPORT GetEmptyBitmap(
+    MemoryPool* pool, int32_t length, std::shared_ptr<MutableBuffer>* result);
+
 }  // namespace arrow
 
 #endif

cpp/src/arrow/column-test.cc

@@ -22,6 +22,7 @@
 
 #include "gtest/gtest.h"
 
+#include "arrow/array.h"
 #include "arrow/column.h"
 #include "arrow/schema.h"
 #include "arrow/test-util.h"

cpp/src/arrow/io/hdfs.cc

@@ -289,13 +289,9 @@ class HdfsClient::HdfsClientImpl {
 
     // connect to HDFS with the builder object
     hdfsBuilder* builder = hdfsNewBuilder();
-    if (!config->host.empty()) {
-      hdfsBuilderSetNameNode(builder, config->host.c_str());
-    }
+    if (!config->host.empty()) { hdfsBuilderSetNameNode(builder, config->host.c_str()); }
     hdfsBuilderSetNameNodePort(builder, config->port);
-    if (!config->user.empty()) {
-      hdfsBuilderSetUserName(builder, config->user.c_str());
-    }
+    if (!config->user.empty()) { hdfsBuilderSetUserName(builder, config->user.c_str()); }
     if (!config->kerb_ticket.empty()) {
       hdfsBuilderSetKerbTicketCachePath(builder, config->kerb_ticket.c_str());
     }

cpp/src/arrow/io/libhdfs_shim.cc

@@ -74,12 +74,9 @@ static HINSTANCE libjvm_handle = NULL;
 // NOTE(wesm): cpplint does not like use of short and other imprecise C types
 
 static hdfsBuilder* (*ptr_hdfsNewBuilder)(void) = NULL;
-static void (*ptr_hdfsBuilderSetNameNode)(
-    hdfsBuilder* bld, const char* nn) = NULL;
-static void (*ptr_hdfsBuilderSetNameNodePort)(
-    hdfsBuilder* bld, tPort port) = NULL;
-static void (*ptr_hdfsBuilderSetUserName)(
-    hdfsBuilder* bld, const char* userName) = NULL;
+static void (*ptr_hdfsBuilderSetNameNode)(hdfsBuilder* bld, const char* nn) = NULL;
+static void (*ptr_hdfsBuilderSetNameNodePort)(hdfsBuilder* bld, tPort port) = NULL;
+static void (*ptr_hdfsBuilderSetUserName)(hdfsBuilder* bld, const char* userName) = NULL;
 static void (*ptr_hdfsBuilderSetKerbTicketCachePath)(
     hdfsBuilder* bld, const char* kerbTicketCachePath) = NULL;
 static hdfsFS (*ptr_hdfsBuilderConnect)(hdfsBuilder* bld) = NULL;
@@ -173,9 +170,9 @@ void hdfsBuilderSetUserName(hdfsBuilder* bld, const char* userName) {
   ptr_hdfsBuilderSetUserName(bld, userName);
 }
 
-void hdfsBuilderSetKerbTicketCachePath(hdfsBuilder* bld,
-    const char* kerbTicketCachePath) {
-  ptr_hdfsBuilderSetKerbTicketCachePath(bld , kerbTicketCachePath);
+void hdfsBuilderSetKerbTicketCachePath(
+    hdfsBuilder* bld, const char* kerbTicketCachePath) {
+  ptr_hdfsBuilderSetKerbTicketCachePath(bld, kerbTicketCachePath);
 }
 
 hdfsFS hdfsBuilderConnect(hdfsBuilder* bld) {
@@ -364,7 +361,7 @@ static std::vector<fs::path> get_potential_libhdfs_paths() {
   std::vector<fs::path> libhdfs_potential_paths;
   std::string file_name;
 
-  // OS-specific file name
+// OS-specific file name
 #ifdef __WIN32
   file_name = "hdfs.dll";
 #elif __APPLE__
@@ -374,10 +371,7 @@ static std::vector<fs::path> get_potential_libhdfs_paths() {
 #endif
 
   // Common paths
-  std::vector<fs::path> search_paths = {
-      fs::path(""),
-      fs::path(".")
-  };
+  std::vector<fs::path> search_paths = {fs::path(""), fs::path(".")};
 
   // Path from environment variable
   const char* hadoop_home = std::getenv("HADOOP_HOME");
@@ -387,9 +381,7 @@ static std::vector<fs::path> get_potential_libhdfs_paths() {
   }
 
   const char* libhdfs_dir = std::getenv("ARROW_LIBHDFS_DIR");
-  if (libhdfs_dir != nullptr) {
-    search_paths.push_back(fs::path(libhdfs_dir));
-  }
+  if (libhdfs_dir != nullptr) { search_paths.push_back(fs::path(libhdfs_dir)); }
 
   // All paths with file name
   for (auto& path : search_paths) {

cpp/src/arrow/ipc/CMakeLists.txt

@@ -34,6 +34,8 @@ set(ARROW_IPC_TEST_LINK_LIBS
 set(ARROW_IPC_SRCS
   adapter.cc
   file.cc
+  json.cc
+  json-internal.cc
   metadata.cc
   metadata-internal.cc
 )
@@ -79,6 +81,10 @@ ADD_ARROW_TEST(ipc-metadata-test)
 ARROW_TEST_LINK_LIBRARIES(ipc-metadata-test
   ${ARROW_IPC_TEST_LINK_LIBS})
 
+ADD_ARROW_TEST(ipc-json-test)
+ARROW_TEST_LINK_LIBRARIES(ipc-json-test
+  ${ARROW_IPC_TEST_LINK_LIBS})
+
 # make clean will delete the generated file
 set_source_files_properties(Metadata_generated.h PROPERTIES GENERATED TRUE)
 
@@ -114,6 +120,7 @@ add_dependencies(arrow_objlib metadata_fbs)
 install(FILES
   adapter.h
   file.h
+  json.h
   metadata.h
   DESTINATION include/arrow/ipc)
 

cpp/src/arrow/ipc/adapter.cc

@@ -106,7 +106,7 @@ Status VisitArray(const Array* arr, std::vector<flatbuf::FieldNode>* field_nodes
     buffers->push_back(binary_arr->data());
   } else if (arr->type_enum() == Type::LIST) {
     const auto list_arr = static_cast<const ListArray*>(arr);
-    buffers->push_back(list_arr->offset_buffer());
+    buffers->push_back(list_arr->offsets());
     RETURN_NOT_OK(VisitArray(
         list_arr->values().get(), field_nodes, buffers, max_recursion_depth - 1));
   } else if (arr->type_enum() == Type::STRUCT) {