Skip to content

Revert .oindex and .vindex additions in _ElementwiseFunctionArray, NativeEndiannessArray, and BoolTypeArray classes #8921

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 13 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 11 additions & 6 deletions xarray/coding/strings.py
Original file line number Diff line number Diff line change
Expand Up @@ -249,15 +249,20 @@ def shape(self) -> tuple[int, ...]:
def __repr__(self):
return f"{type(self).__name__}({self.array!r})"

def _vindex_get(self, key):
return _numpy_char_to_bytes(self.array.vindex[key])

def _oindex_get(self, key):
return _numpy_char_to_bytes(self.array.oindex[key])
def _check_and_raise_if_non_basic_indexer(self, indexer) -> None:
...
# TODO: this is a temporary fix until BackendArray supports vindex and oindex

def __getitem__(self, key):
# require slicing the last dimension completely
key = type(key)(indexing.expanded_indexer(key.tuple, self.array.ndim))
if key.tuple[-1] != slice(None):
raise IndexError("too many indices")
return _numpy_char_to_bytes(self.array[key])
# TODO: this is a temporary fix until BackendArray supports vindex and oindex
if isinstance(key, indexing.OuterIndexer):
data = self.array.oindex[key]
elif isinstance(key, indexing.VectorizedIndexer):
data = self.array.vindex[key]
else:
data = self.array[key]
return _numpy_char_to_bytes(data)
51 changes: 33 additions & 18 deletions xarray/coding/variables.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,14 +68,19 @@ def __init__(self, array, func: Callable, dtype: np.typing.DTypeLike):
def dtype(self) -> np.dtype:
return np.dtype(self._dtype)

def _oindex_get(self, key):
return type(self)(self.array.oindex[key], self.func, self.dtype)

def _vindex_get(self, key):
return type(self)(self.array.vindex[key], self.func, self.dtype)
def _check_and_raise_if_non_basic_indexer(self, indexer) -> None:
...
# TODO: this is a temporary fix until BackendArray supports vindex and oindex

def __getitem__(self, key):
return type(self)(self.array[key], self.func, self.dtype)
# TODO: this is a temporary fix until BackendArray supports vindex and oindex
if isinstance(key, indexing.OuterIndexer):
data = self.array.oindex[key]
elif isinstance(key, indexing.VectorizedIndexer):
data = self.array.vindex[key]
else:
data = self.array[key]
return type(self)(data, self.func, self.dtype)

def get_duck_array(self):
return self.func(self.array.get_duck_array())
Expand Down Expand Up @@ -113,14 +118,19 @@ def __init__(self, array) -> None:
def dtype(self) -> np.dtype:
return np.dtype(self.array.dtype.kind + str(self.array.dtype.itemsize))

def _oindex_get(self, key):
return np.asarray(self.array.oindex[key], dtype=self.dtype)

def _vindex_get(self, key):
return np.asarray(self.array.vindex[key], dtype=self.dtype)
def _check_and_raise_if_non_basic_indexer(self, indexer) -> None:
...
# TODO: this is a temporary fix until BackendArray supports vindex and oindex

def __getitem__(self, key) -> np.ndarray:
return np.asarray(self.array[key], dtype=self.dtype)
# TODO: this is a temporary fix until BackendArray supports vindex and oindex
if isinstance(key, indexing.OuterIndexer):
data = self.array.oindex[key]
elif isinstance(key, indexing.VectorizedIndexer):
data = self.array.vindex[key]
else:
data = self.array[key]
return np.asarray(data, dtype=self.dtype)


class BoolTypeArray(indexing.ExplicitlyIndexedNDArrayMixin):
Expand Down Expand Up @@ -151,14 +161,19 @@ def __init__(self, array) -> None:
def dtype(self) -> np.dtype:
return np.dtype("bool")

def _oindex_get(self, key):
return np.asarray(self.array.oindex[key], dtype=self.dtype)

def _vindex_get(self, key):
return np.asarray(self.array.vindex[key], dtype=self.dtype)
def _check_and_raise_if_non_basic_indexer(self, indexer) -> None:
...
# TODO: this is a temporary fix until BackendArray supports vindex and oindex

def __getitem__(self, key) -> np.ndarray:
return np.asarray(self.array[key], dtype=self.dtype)
# TODO: this is a temporary fix until BackendArray supports vindex and oindex
if isinstance(key, indexing.OuterIndexer):
data = self.array.oindex[key]
elif isinstance(key, indexing.VectorizedIndexer):
data = self.array.vindex[key]
else:
data = self.array[key]
return np.asarray(data, dtype=self.dtype)


def lazy_elemwise_func(array, func: Callable, dtype: np.typing.DTypeLike):
Expand Down
26 changes: 24 additions & 2 deletions xarray/core/indexing.py
Original file line number Diff line number Diff line change
Expand Up @@ -639,7 +639,18 @@ def shape(self) -> _Shape:
return tuple(shape)

def get_duck_array(self):
if isinstance(self.array, ExplicitlyIndexedNDArrayMixin):
from xarray.coding import strings, variables

if isinstance(self.array, ExplicitlyIndexedNDArrayMixin) and not isinstance(
self.array,
(
strings.StackedBytesArray,
variables._ElementwiseFunctionArray,
variables.BoolTypeArray,
variables.NativeEndiannessArray,
),
):
# TODO: Remove the isinstance check for variables.BoolTypeArray and variables.NativeEndiannessArray once the BackendArrray is updated with oindex and vindex properties
array = apply_indexer(self.array, self.key)
else:
# If the array is not an ExplicitlyIndexedNDArrayMixin,
Expand Down Expand Up @@ -715,7 +726,18 @@ def shape(self) -> _Shape:
return np.broadcast(*self.key.tuple).shape

def get_duck_array(self):
if isinstance(self.array, ExplicitlyIndexedNDArrayMixin):
from xarray.coding import strings, variables

if isinstance(self.array, ExplicitlyIndexedNDArrayMixin) and not isinstance(
self.array,
(
strings.StackedBytesArray,
variables._ElementwiseFunctionArray,
variables.BoolTypeArray,
variables.NativeEndiannessArray,
),
):
# TODO: Remove the isinstance check for variables.BoolTypeArray and variables.NativeEndiannessArray once the BackendArrray is updated with oindex and vindex properties
array = apply_indexer(self.array, self.key)
else:
# If the array is not an ExplicitlyIndexedNDArrayMixin,
Expand Down
9 changes: 9 additions & 0 deletions xarray/tests/test_backends.py
Original file line number Diff line number Diff line change
Expand Up @@ -3211,6 +3211,15 @@
unpickled = pickle.loads(pickle.dumps(ds))
assert_identical(unpickled, data)

def test_scipy_wrapper_array_oindex_vindex(self) -> None:
ds = xr.Dataset()
ds["A"] = xr.DataArray([[1, "a"], [2, "b"]], dims=["x", "y"])
with create_tmp_file(allow_cleanup_failure=False) as path:
ds.to_netcdf(path, engine="scipy")
with xr.open_dataset(path, engine="scipy") as ds2:
with create_tmp_file(allow_cleanup_failure=False) as path2:
ds2.sel(y=[1]).to_netcdf(path2)

Check failure on line 3221 in xarray/tests/test_backends.py

View workflow job for this annotation

GitHub Actions / ubuntu-latest py3.9

TestScipyInMemoryData.test_scipy_wrapper_array_oindex_vindex NotImplementedError: StackedBytesArray._oindex_get method should be overridden

Check failure on line 3221 in xarray/tests/test_backends.py

View workflow job for this annotation

GitHub Actions / ubuntu-latest py3.12

TestScipyInMemoryData.test_scipy_wrapper_array_oindex_vindex NotImplementedError: StackedBytesArray._oindex_get method should be overridden

Check failure on line 3221 in xarray/tests/test_backends.py

View workflow job for this annotation

GitHub Actions / macos-latest py3.9

TestScipyInMemoryData.test_scipy_wrapper_array_oindex_vindex NotImplementedError: StackedBytesArray._oindex_get method should be overridden

Check failure on line 3221 in xarray/tests/test_backends.py

View workflow job for this annotation

GitHub Actions / macos-latest py3.12

TestScipyInMemoryData.test_scipy_wrapper_array_oindex_vindex NotImplementedError: StackedBytesArray._oindex_get method should be overridden

Check failure on line 3221 in xarray/tests/test_backends.py

View workflow job for this annotation

GitHub Actions / ubuntu-latest py3.9 min-all-deps

TestScipyInMemoryData.test_scipy_wrapper_array_oindex_vindex NotImplementedError: StackedBytesArray._oindex_get method should be overridden

Check failure on line 3221 in xarray/tests/test_backends.py

View workflow job for this annotation

GitHub Actions / ubuntu-latest py3.11 all-but-dask

TestScipyInMemoryData.test_scipy_wrapper_array_oindex_vindex NotImplementedError: StackedBytesArray._oindex_get method should be overridden


@requires_scipy
class TestScipyFileObject(CFEncodedBase, NetCDF3Only):
Expand Down
2 changes: 1 addition & 1 deletion xarray/tests/test_coding_strings.py
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,7 @@ def test_StackedBytesArray_vectorized_indexing() -> None:

V = IndexerMaker(indexing.VectorizedIndexer)
indexer = V[np.array([[0, 1], [1, 0]])]
actual = stacked.vindex[indexer]
actual = stacked[indexer]
assert_array_equal(actual, expected)


Expand Down
Loading