Gross hack to avoid unwanted nanosecond promotion with tz-aware timestamp types

Author: wesm

cpp/src/arrow/python/arrow_to_pandas.cc

@@ -619,13 +619,18 @@ inline Status ConvertStruct(const PandasOptions& options, const ChunkedArray& da
   auto array_type = arr->type();
   std::vector<OwnedRef> fields_data(num_fields);
   OwnedRef dict_item;
+
+  // XXX(wesm): In ARROW-7723, we found as a result of ARROW-3789
+  PandasOptions modified_options = options;
+  modified_options.ignore_timezone = true;
+
   for (int c = 0; c < data.num_chunks(); c++) {
     auto arr = checked_cast<const StructArray*>(data.chunk(c).get());
     // Convert the struct arrays first
     for (int32_t i = 0; i < num_fields; i++) {
       PyObject* numpy_array;
-      RETURN_NOT_OK(ConvertArrayToPandas(options, arr->field(static_cast<int>(i)),
-                                         nullptr, &numpy_array));
+      RETURN_NOT_OK(ConvertArrayToPandas(
+          modified_options, arr->field(static_cast<int>(i)), nullptr, &numpy_array));
       fields_data[i].reset(numpy_array);
     }
 
@@ -1678,7 +1683,8 @@ static Status GetPandasWriterType(const ChunkedArray& data, const PandasOptions&
       break;
     case Type::TIMESTAMP: {
       const auto& ts_type = checked_cast<const TimestampType&>(*data.type());
-      if (ts_type.timezone() != "") {
+      // XXX: Hack here for ARROW-7723
+      if (ts_type.timezone() != "" && !options.ignore_timezone) {
         *output_type = PandasWriter::DATETIME_NANO_TZ;
       } else if (options.coerce_temporal_nanoseconds) {
         *output_type = PandasWriter::DATETIME_NANO;

cpp/src/arrow/python/arrow_to_pandas.h

@@ -55,6 +55,10 @@ struct PandasOptions {
   /// Coerce all date and timestamp to datetime64[ns]
   bool coerce_temporal_nanoseconds = false;
 
+  /// XXX(wesm): Hack for ARROW-7723 to opt out of DATETIME_NANO_TZ conversion
+  /// path
+  bool ignore_timezone = false;
+
   /// \brief If true, do not create duplicate PyObject versions of equal
   /// objects. This only applies to immutable objects like strings or datetime
   /// objects

python/pyarrow/tests/test_pandas.py

@@ -3215,7 +3215,9 @@ def test_struct_with_timestamp_tz():
     # ARROW-7723
     ts = pd.Timestamp.now()
 
-    for unit in ['s', 'ms', 'us', 'ns']:
+    # XXX: Ensure that this data does not get promoted to nanoseconds (and thus
+    # integers) to preserve behavior in 0.15.1
+    for unit in ['s', 'ms', 'us']:
         arr = pa.array([ts], type=pa.timestamp(unit))
         arr2 = pa.array([ts], type=pa.timestamp(unit, tz='America/New_York'))