Merge remote-tracking branch 'upstream/master' into arrow-string-array-dtype

Author: simonjayhawkins

.travis.yml

@@ -45,7 +45,6 @@ before_install:
   - echo "before_install"
   # Use blocking IO on travis.  Ref:  https://github.com/travis-ci/travis-ci/issues/8920#issuecomment-352661024
   - python -c 'import os,sys,fcntl; flags = fcntl.fcntl(sys.stdout, fcntl.F_GETFL); fcntl.fcntl(sys.stdout, fcntl.F_SETFL, flags&~os.O_NONBLOCK);'
-  - source ci/travis_process_gbq_encryption.sh
   - export PATH="$HOME/miniconda3/bin:$PATH"
   - df -h
   - pwd

ci/travis_encrypt_gbq.sh

@@ -341,6 +341,7 @@ Other enhancements
 - Add keyword ``sort`` to :func:`pivot_table` to allow non-sorting of the result (:issue:`39143`)
 - Add keyword ``dropna`` to :meth:`DataFrame.value_counts` to allow counting rows that include ``NA`` values (:issue:`41325`)
 - :meth:`Series.replace` will now cast results to ``PeriodDtype`` where possible instead of ``object`` dtype (:issue:`41526`)
+- Improved error message in ``corr` and ``cov`` methods on :class:`.Rolling`, :class:`.Expanding`, and :class:`.ExponentialMovingWindow` when ``other`` is not a :class:`DataFrame` or :class:`Series` (:issue:`41741`)
 
 .. ---------------------------------------------------------------------------
 
@@ -804,11 +805,12 @@ Deprecations
 - Deprecated passing arguments (apart from ``value``) as positional in :meth:`DataFrame.fillna` and :meth:`Series.fillna` (:issue:`41485`)
 - Deprecated passing arguments as positional in :meth:`DataFrame.reset_index` (other than ``"level"``) and :meth:`Series.reset_index` (:issue:`41485`)
 - Deprecated construction of :class:`Series` or :class:`DataFrame` with ``DatetimeTZDtype`` data and ``datetime64[ns]`` dtype.  Use ``Series(data).dt.tz_localize(None)`` instead (:issue:`41555`,:issue:`33401`)
+- In a future version, constructing :class:`Series` or :class:`DataFrame` with ``datetime64[ns]`` data and ``DatetimeTZDtype`` will treat the data as wall-times instead of as UTC times (matching DatetimeIndex behavior). To treat the data as UTC times, use ``pd.Series(data).dt.tz_localize("UTC").dt.tz_convert(dtype.tz)`` or ``pd.Series(data.view("int64"), dtype=dtype)`` (:issue:`33401`)
 - Deprecated passing arguments as positional in :meth:`DataFrame.set_axis` and :meth:`Series.set_axis` (other than ``"labels"``) (:issue:`41485`)
 - Deprecated passing arguments as positional in :meth:`DataFrame.where` and :meth:`Series.where` (other than ``"cond"`` and ``"other"``) (:issue:`41485`)
 - Deprecated passing arguments as positional (other than ``filepath_or_buffer``) in :func:`read_csv` (:issue:`41485`)
 - Deprecated passing arguments as positional in :meth:`DataFrame.drop` (other than ``"labels"``) and :meth:`Series.drop` (:issue:`41485`)
--
+- Deprecated passing arguments as positional (other than ``filepath_or_buffer``) in :func:`read_table` (:issue:`41485`)
 
 
 .. _whatsnew_130.deprecations.nuisance_columns:

ci/travis_gbq.json.enc

@@ -66,6 +66,11 @@
     MultiIndex,
 )
 
+# Until https://github.com/numpy/numpy/issues/19078 is sorted out, just suppress
+suppress_npdev_promotion_warning = pytest.mark.filterwarnings(
+    "ignore:Promotion of numbers and bools:FutureWarning"
+)
+
 # ----------------------------------------------------------------
 # Configuration / Settings
 # ----------------------------------------------------------------
@@ -112,6 +117,8 @@ def pytest_collection_modifyitems(items):
         if "/frame/" in item.nodeid:
             item.add_marker(pytest.mark.arraymanager)
 
+        item.add_marker(suppress_npdev_promotion_warning)
+
 
 # Hypothesis
 hypothesis.settings.register_profile(

ci/travis_gbq_config.txt

@@ -32,7 +32,6 @@
 from pandas.compat.numpy import function as nv
 from pandas.util._decorators import Appender
 
-from pandas.core.dtypes.cast import maybe_convert_platform
 from pandas.core.dtypes.common import (
     is_categorical_dtype,
     is_datetime64_dtype,
@@ -1650,4 +1649,6 @@ def _maybe_convert_platform_interval(values) -> ArrayLike:
     else:
         values = extract_array(values, extract_numpy=True)
 
-    return maybe_convert_platform(values)
+    if not hasattr(values, "dtype"):
+        return np.asarray(values)
+    return values

ci/travis_process_gbq_encryption.sh

@@ -19,17 +19,19 @@
     Scalar,
 )
 from pandas.compat import (
+    pa_version_under1p0,
     pa_version_under2p0,
     pa_version_under3p0,
     pa_version_under4p0,
 )
-from pandas.compat.pyarrow import pa_version_under1p0
 from pandas.util._decorators import doc
 from pandas.util._validators import validate_fillna_kwargs
 
+from pandas.core.dtypes.base import ExtensionDtype
 from pandas.core.dtypes.common import (
     is_array_like,
     is_bool_dtype,
+    is_dtype_equal,
     is_integer,
     is_integer_dtype,
     is_object_dtype,
@@ -44,38 +46,41 @@
 from pandas.core.arrays.base import ExtensionArray
 from pandas.core.arrays.boolean import BooleanDtype
 from pandas.core.arrays.integer import Int64Dtype
+from pandas.core.arrays.numeric import NumericDtype
 from pandas.core.arrays.string_ import StringDtype
 from pandas.core.indexers import (
     check_array_indexer,
     validate_indices,
 )
 from pandas.core.strings.object_array import ObjectStringArrayMixin
 
-try:
+# PyArrow backed StringArrays are available starting at 1.0.0, but this
+# file is imported from even if pyarrow is < 1.0.0, before pyarrow.compute
+# and its compute functions existed. GH38801
+if not pa_version_under1p0:
     import pyarrow as pa
-except ImportError:
-    pa = None
-else:
-    # PyArrow backed StringArrays are available starting at 1.0.0, but this
-    # file is imported from even if pyarrow is < 1.0.0, before pyarrow.compute
-    # and its compute functions existed. GH38801
-    if not pa_version_under1p0:
-        import pyarrow.compute as pc
-
-        ARROW_CMP_FUNCS = {
-            "eq": pc.equal,
-            "ne": pc.not_equal,
-            "lt": pc.less,
-            "gt": pc.greater,
-            "le": pc.less_equal,
-            "ge": pc.greater_equal,
-        }
+    import pyarrow.compute as pc
+
+    ARROW_CMP_FUNCS = {
+        "eq": pc.equal,
+        "ne": pc.not_equal,
+        "lt": pc.less,
+        "gt": pc.greater,
+        "le": pc.less_equal,
+        "ge": pc.greater_equal,
+    }
 
 
 if TYPE_CHECKING:
     from pandas import Series
 
 
+def _chk_pyarrow_available() -> None:
+    if pa_version_under1p0:
+        msg = "pyarrow>=1.0.0 is required for PyArrow backed StringArray."
+        raise ImportError(msg)
+
+
 # TODO: Inherit directly from BaseStringArrayMethods. Currently we inherit from
 # ObjectStringArrayMixin because we want to have the object-dtype based methods as
 # fallback for the ones that pyarrow doesn't yet support
@@ -125,10 +130,8 @@ class ArrowStringArray(OpsMixin, ExtensionArray, ObjectStringArrayMixin):
     Length: 4, dtype: string[pyarrow]
     """
 
-    _dtype = StringDtype(storage="pyarrow")
-
     def __init__(self, values):
-        self._chk_pyarrow_available()
+        self._dtype = StringDtype(storage="pyarrow")
         if isinstance(values, pa.Array):
             self._data = pa.chunked_array([values])
         elif isinstance(values, pa.ChunkedArray):
@@ -141,19 +144,11 @@ def __init__(self, values):
                 "ArrowStringArray requires a PyArrow (chunked) array of string type"
             )
 
-    @classmethod
-    def _chk_pyarrow_available(cls) -> None:
-        # TODO: maybe update import_optional_dependency to allow a minimum
-        # version to be specified rather than use the global minimum
-        if pa is None or pa_version_under1p0:
-            msg = "pyarrow>=1.0.0 is required for PyArrow backed StringArray."
-            raise ImportError(msg)
-
     @classmethod
     def _from_sequence(cls, scalars, dtype: Dtype | None = None, copy: bool = False):
         from pandas.core.arrays.masked import BaseMaskedArray
 
-        cls._chk_pyarrow_available()
+        _chk_pyarrow_available()
 
         if dtype and not (isinstance(dtype, str) and dtype == "string"):
             dtype = pandas_dtype(dtype)
@@ -206,10 +201,14 @@ def to_numpy(  # type: ignore[override]
         """
         # TODO: copy argument is ignored
 
-        if na_value is lib.no_default:
-            na_value = self._dtype.na_value
-        result = self._data.__array__(dtype=dtype)
-        result[isna(result)] = na_value
+        result = np.array(self._data, dtype=dtype)
+        if self._data.null_count > 0:
+            if na_value is lib.no_default:
+                if dtype and np.issubdtype(dtype, np.floating):
+                    return result
+                na_value = self._dtype.na_value
+            mask = self.isna()
+            result[mask] = na_value
         return result
 
     def __len__(self) -> int:
@@ -653,6 +652,24 @@ def value_counts(self, dropna: bool = True) -> Series:
 
         return Series(counts, index=index).astype("Int64")
 
+    def astype(self, dtype, copy=True):
+        dtype = pandas_dtype(dtype)
+
+        if is_dtype_equal(dtype, self.dtype):
+            if copy:
+                return self.copy()
+            return self
+
+        elif isinstance(dtype, NumericDtype):
+            data = self._data.cast(pa.from_numpy_dtype(dtype.numpy_dtype))
+            return dtype.__from_arrow__(data)
+
+        elif isinstance(dtype, ExtensionDtype):
+            cls = dtype.construct_array_type()
+            return cls._from_sequence(self, dtype=dtype, copy=copy)
+
+        return super().astype(dtype, copy)
+
     # ------------------------------------------------------------------------
     # String methods interface
 

doc/README.rst

@@ -1657,6 +1657,22 @@ def maybe_cast_to_datetime(
                             # Numeric values are UTC at this point,
                             # so localize and convert
                             # equiv: Series(dta).astype(dtype) # though deprecated
+                            if getattr(vdtype, "kind", None) == "M":
+                                # GH#24559, GH#33401 deprecate behavior inconsistent
+                                #  with DatetimeArray/DatetimeIndex
+                                warnings.warn(
+                                    "In a future version, constructing a Series "
+                                    "from datetime64[ns] data and a "
+                                    "DatetimeTZDtype will interpret the data "
+                                    "as wall-times instead of "
+                                    "UTC times, matching the behavior of "
+                                    "DatetimeIndex. To treat the data as UTC "
+                                    "times, use pd.Series(data).dt"
+                                    ".tz_localize('UTC').tz_convert(dtype.tz) "
+                                    "or pd.Series(data.view('int64'), dtype=dtype)",
+                                    FutureWarning,
+                                    stacklevel=5,
+                                )
 
                             value = dta.tz_localize("UTC").tz_convert(dtype.tz)
                 except OutOfBoundsDatetime:

doc/source/whatsnew/v1.3.0.rst

@@ -537,9 +537,6 @@ def _prep_ndarray(values, copy: bool = True) -> np.ndarray:
         def convert(v):
             if not is_list_like(v) or isinstance(v, ABCDataFrame):
                 return v
-            elif not hasattr(v, "dtype") and not isinstance(v, (list, tuple, range)):
-                # TODO: should we cast these to list?
-                return v
 
             v = extract_array(v, extract_numpy=True)
             res = maybe_convert_platform(v)