ARROW-392: [C++/Java] String IPC integration testing / fixes. Add array / record batch pretty-printing

cpp/CMakeLists.txt

@@ -745,6 +745,7 @@ set(ARROW_SRCS
   src/arrow/array.cc
   src/arrow/builder.cc
   src/arrow/column.cc
+  src/arrow/pretty_print.cc
   src/arrow/schema.cc
   src/arrow/table.cc
   src/arrow/type.cc

cpp/src/arrow/CMakeLists.txt

@@ -21,6 +21,7 @@ install(FILES
   array.h
   column.h
   builder.h
+  pretty_print.h
   schema.h
   table.h
   type.h
@@ -37,6 +38,7 @@ set(ARROW_TEST_LINK_LIBS ${ARROW_MIN_TEST_LIBS})
 
 ADD_ARROW_TEST(array-test)
 ADD_ARROW_TEST(column-test)
+ADD_ARROW_TEST(pretty_print-test)
 ADD_ARROW_TEST(schema-test)
 ADD_ARROW_TEST(table-test)
 

cpp/src/arrow/ipc/ipc-json-test.cc

@@ -96,26 +96,6 @@ void CheckPrimitive(const std::shared_ptr<DataType>& type,
   TestArrayRoundTrip(*array.get());
 }
 
-template <typename TYPE, typename C_TYPE>
-void MakeArray(const std::shared_ptr<DataType>& type, const std::vector<bool>& is_valid,
-    const std::vector<C_TYPE>& values, std::shared_ptr<Array>* out) {
-  std::shared_ptr<Buffer> values_buffer;
-  std::shared_ptr<Buffer> values_bitmap;
-
-  ASSERT_OK(test::CopyBufferFromVector(values, &values_buffer));
-  ASSERT_OK(test::GetBitmapFromBoolVector(is_valid, &values_bitmap));
-
-  using ArrayType = typename TypeTraits<TYPE>::ArrayType;
-
-  int32_t null_count = 0;
-  for (bool val : is_valid) {
-    if (!val) { ++null_count; }
-  }
-
-  *out = std::make_shared<ArrayType>(type, static_cast<int32_t>(values.size()),
-      values_buffer, null_count, values_bitmap);
-}
-
 TEST(TestJsonSchemaWriter, FlatTypes) {
   std::vector<std::shared_ptr<Field>> fields = {field("f0", int8()),
       field("f1", int16(), false), field("f2", int32()), field("f3", int64(), false),

cpp/src/arrow/ipc/json-integration-test.cc

@@ -31,6 +31,7 @@
 #include "arrow/io/file.h"
 #include "arrow/ipc/file.h"
 #include "arrow/ipc/json.h"
+#include "arrow/pretty_print.h"
 #include "arrow/schema.h"
 #include "arrow/table.h"
 #include "arrow/test-util.h"
@@ -171,6 +172,12 @@ static Status ValidateArrowVsJson(
     if (!json_batch->Equals(*arrow_batch.get())) {
       std::stringstream ss;
       ss << "Record batch " << i << " did not match";
+
+      ss << "\nJSON: \n ";
+      RETURN_NOT_OK(PrettyPrint(*json_batch.get(), &ss));
+
+      ss << "\nArrow: \n ";
+      RETURN_NOT_OK(PrettyPrint(*arrow_batch.get(), &ss));
       return Status::Invalid(ss.str());
     }
   }

cpp/src/arrow/ipc/json-internal.cc

@@ -343,7 +343,7 @@ class JsonSchemaWriter : public TypeVisitor {
 
 class JsonArrayWriter : public ArrayVisitor {
  public:
-  explicit JsonArrayWriter(const std::string& name, const Array& array, RjWriter* writer)
+  JsonArrayWriter(const std::string& name, const Array& array, RjWriter* writer)
       : name_(name), array_(array), writer_(writer) {}
 
   Status Write() { return VisitArray(name_, array_); }

cpp/src/arrow/pretty_print-test.cc

@@ -0,0 +1,87 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <memory>
+#include <sstream>
+#include <vector>
+
+#include "gtest/gtest.h"
+
+#include "arrow/array.h"
+#include "arrow/pretty_print.h"
+#include "arrow/test-util.h"
+#include "arrow/type.h"
+#include "arrow/type_traits.h"
+#include "arrow/types/list.h"
+#include "arrow/types/primitive.h"
+#include "arrow/types/string.h"
+#include "arrow/types/struct.h"
+
+namespace arrow {
+
+class TestArrayPrinter : public ::testing::Test {
+ public:
+  void SetUp() {}
+
+  void Print(const Array& array) {}
+
+ private:
+  std::ostringstream sink_;
+};
+
+template <typename TYPE, typename C_TYPE>
+void CheckPrimitive(const std::vector<bool>& is_valid, const std::vector<C_TYPE>& values,
+    const char* expected) {
+  std::ostringstream sink;
+
+  MemoryPool* pool = default_memory_pool();
+  typename TypeTraits<TYPE>::BuilderType builder(pool, std::make_shared<TYPE>());
+
+  for (size_t i = 0; i < values.size(); ++i) {
+    if (is_valid[i]) {
+      ASSERT_OK(builder.Append(values[i]));
+    } else {
+      ASSERT_OK(builder.AppendNull());
+    }
+  }
+
+  std::shared_ptr<Array> array;
+  ASSERT_OK(builder.Finish(&array));
+
+  ASSERT_OK(PrettyPrint(*array.get(), &sink));
+
+  std::string result = sink.str();
+  ASSERT_EQ(std::string(expected, strlen(expected)), result);
+}
+
+TEST_F(TestArrayPrinter, PrimitiveType) {
+  std::vector<bool> is_valid = {true, true, false, true, false};
+
+  std::vector<int32_t> values = {0, 1, 2, 3, 4};
+  static const char* expected = R"expected([0, 1, null, 3, null])expected";
+  CheckPrimitive<Int32Type, int32_t>(is_valid, values, expected);
+
+  std::vector<std::string> values2 = {"foo", "bar", "", "baz", ""};
+  static const char* ex2 = R"expected(["foo", "bar", null, "baz", null])expected";
+  CheckPrimitive<StringType, std::string>(is_valid, values2, ex2);
+}
+
+}  // namespace arrow

cpp/src/arrow/pretty_print.cc

@@ -0,0 +1,192 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <ostream>
+#include <string>
+
+#include "arrow/array.h"
+#include "arrow/pretty_print.h"
+#include "arrow/table.h"
+#include "arrow/type.h"
+#include "arrow/type_traits.h"
+#include "arrow/types/list.h"
+#include "arrow/types/string.h"
+#include "arrow/types/struct.h"
+#include "arrow/util/status.h"
+
+namespace arrow {
+
+class ArrayPrinter : public ArrayVisitor {
+ public:
+  ArrayPrinter(const Array& array, std::ostream* sink) : array_(array), sink_(sink) {}
+
+  Status Print() { return VisitArray(array_); }
+
+  Status VisitArray(const Array& array) { return array.Accept(this); }
+
+  template <typename T>
+  typename std::enable_if<IsNumeric<T>::value, void>::type WriteDataValues(
+      const T& array) {
+    const auto data = array.raw_data();
+    for (int i = 0; i < array.length(); ++i) {
+      if (i > 0) { (*sink_) << ", "; }
+      if (array.IsNull(i)) {
+        (*sink_) << "null";
+      } else {
+        (*sink_) << data[i];
+      }
+    }
+  }
+
+  // String (Utf8), Binary
+  template <typename T>
+  typename std::enable_if<std::is_base_of<BinaryArray, T>::value, void>::type
+  WriteDataValues(const T& array) {
+    int32_t length;
+    for (int i = 0; i < array.length(); ++i) {
+      if (i > 0) { (*sink_) << ", "; }
+      if (array.IsNull(i)) {
+        (*sink_) << "null";
+      } else {
+        const char* buf = reinterpret_cast<const char*>(array.GetValue(i, &length));
+        (*sink_) << "\"" << std::string(buf, length) << "\"";
+      }
+    }
+  }
+
+  template <typename T>
+  typename std::enable_if<std::is_base_of<BooleanArray, T>::value, void>::type
+  WriteDataValues(const T& array) {
+    for (int i = 0; i < array.length(); ++i) {
+      if (i > 0) { (*sink_) << ", "; }
+      if (array.IsNull(i)) {
+        (*sink_) << "null";
+      } else {
+        (*sink_) << (array.Value(i) ? "true" : "false");
+      }
+    }
+  }
+
+  void OpenArray() { (*sink_) << "["; }
+
+  void CloseArray() { (*sink_) << "]"; }
+
+  template <typename T>
+  Status WritePrimitive(const T& array) {
+    OpenArray();
+    WriteDataValues(array);
+    CloseArray();
+    return Status::OK();
+  }
+
+  template <typename T>
+  Status WriteVarBytes(const T& array) {
+    OpenArray();
+    WriteDataValues(array);
+    CloseArray();
+    return Status::OK();
+  }
+
+  Status Visit(const NullArray& array) override { return Status::OK(); }
+
+  Status Visit(const BooleanArray& array) override { return WritePrimitive(array); }
+
+  Status Visit(const Int8Array& array) override { return WritePrimitive(array); }
+
+  Status Visit(const Int16Array& array) override { return WritePrimitive(array); }
+
+  Status Visit(const Int32Array& array) override { return WritePrimitive(array); }
+
+  Status Visit(const Int64Array& array) override { return WritePrimitive(array); }
+
+  Status Visit(const UInt8Array& array) override { return WritePrimitive(array); }
+
+  Status Visit(const UInt16Array& array) override { return WritePrimitive(array); }
+
+  Status Visit(const UInt32Array& array) override { return WritePrimitive(array); }
+
+  Status Visit(const UInt64Array& array) override { return WritePrimitive(array); }
+
+  Status Visit(const HalfFloatArray& array) override { return WritePrimitive(array); }
+
+  Status Visit(const FloatArray& array) override { return WritePrimitive(array); }
+
+  Status Visit(const DoubleArray& array) override { return WritePrimitive(array); }
+
+  Status Visit(const StringArray& array) override { return WriteVarBytes(array); }
+
+  Status Visit(const BinaryArray& array) override { return WriteVarBytes(array); }
+
+  Status Visit(const DateArray& array) override { return Status::NotImplemented("date"); }
+
+  Status Visit(const TimeArray& array) override { return Status::NotImplemented("time"); }
+
+  Status Visit(const TimestampArray& array) override {
+    return Status::NotImplemented("timestamp");
+  }
+
+  Status Visit(const IntervalArray& array) override {
+    return Status::NotImplemented("interval");
+  }
+
+  Status Visit(const DecimalArray& array) override {
+    return Status::NotImplemented("decimal");
+  }
+
+  Status Visit(const ListArray& array) override {
+    // auto type = static_cast<const ListType*>(array.type().get());
+    // for (size_t i = 0; i < fields.size(); ++i) {
+    //   RETURN_NOT_OK(VisitArray(fields[i]->name, *arrays[i].get()));
+    // }
+    // return WriteChildren(type->children(), {array.values()});
+    return Status::OK();
+  }
+
+  Status Visit(const StructArray& array) override {
+    // auto type = static_cast<const StructType*>(array.type().get());
+    // for (size_t i = 0; i < fields.size(); ++i) {
+    //   RETURN_NOT_OK(VisitArray(fields[i]->name, *arrays[i].get()));
+    // }
+    // return WriteChildren(type->children(), array.fields());
+    return Status::OK();
+  }
+
+  Status Visit(const UnionArray& array) override {
+    return Status::NotImplemented("union");
+  }
+
+ private:
+  const Array& array_;
+  std::ostream* sink_;
+};
+
+Status PrettyPrint(const Array& arr, std::ostream* sink) {
+  ArrayPrinter printer(arr, sink);
+  return printer.Print();
+}
+
+Status PrettyPrint(const RecordBatch& batch, std::ostream* sink) {
+  for (int i = 0; i < batch.num_columns(); ++i) {
+    const std::string& name = batch.column_name(i);
+    (*sink) << name << ": ";
+    RETURN_NOT_OK(PrettyPrint(*batch.column(i).get(), sink));
+    (*sink) << "\n";
+  }
+  return Status::OK();
+}
+
+}  // namespace arrow

cpp/src/arrow/pretty_print.h

@@ -0,0 +1,35 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_PRETTY_PRINT_H
+#define ARROW_PRETTY_PRINT_H
+
+#include <ostream>
+
+#include "arrow/type_fwd.h"
+#include "arrow/util/visibility.h"
+
+namespace arrow {
+
+class Status;
+
+Status ARROW_EXPORT PrettyPrint(const RecordBatch& batch, std::ostream* sink);
+Status ARROW_EXPORT PrettyPrint(const Array& arr, std::ostream* sink);
+
+}  // namespace arrow
+
+#endif  // ARROW_PRETTY_PRINT_H