CFTimeIndex Resampling

xarray/backends/zarr.py

@@ -8,7 +8,7 @@
 from ..core import indexing
 from ..core.pycompat import OrderedDict, integer_types, iteritems
 from ..core.utils import FrozenOrderedDict, HiddenKeyDict
-from .common import AbstractWritableDataStore, ArrayWriter, BackendArray
+from .common import AbstractWritableDataStore, BackendArray
 
 # need some special secret attributes to tell us the dimensions
 _DIMENSION_KEY = '_ARRAY_DIMENSIONS'

xarray/core/common.py

@@ -657,6 +657,7 @@ def resample(self, freq=None, dim=None, how=None, skipna=None,
         """
         # TODO support non-string indexer after removing the old API.
 
+        from ..coding.cftime_offsets import cftime_range
         from .dataarray import DataArray
         from .resample import RESAMPLE_DIM
         from ..coding.cftimeindex import CFTimeIndex
@@ -690,20 +691,35 @@ def resample(self, freq=None, dim=None, how=None, skipna=None,
                             "was passed %r" % dim)
 
         if isinstance(self.indexes[dim_name], CFTimeIndex):
-            raise NotImplementedError(
-                'Resample is currently not supported along a dimension '
-                'indexed by a CFTimeIndex.  For certain kinds of downsampling '
-                'it may be possible to work around this by converting your '
-                'time index to a DatetimeIndex using '
-                'CFTimeIndex.to_datetimeindex.  Use caution when doing this '
-                'however, because switching to a DatetimeIndex from a '
-                'CFTimeIndex with a non-standard calendar entails a change '
-                'in the calendar type, which could lead to subtle and silent '
-                'errors.'
-            )
+            from ..coding.cftime_offsets import to_offset
+            from .resample_cftime import (_get_time_bins, _offset_timedelta,
+                                          _adjust_binner_for_upsample)
+            offset = to_offset(freq)
+            times = self.indexes[dim_name]
+            binner, labels = _get_time_bins(self.indexes[dim_name],
+                                            offset,
+                                            closed, label, base)
+            if times.size > labels.size:
+                # if we're downsampling CFTimeIndex, do this:
+                if closed == 'right':
+                    fill_method = 'bfill'
+                else:
+                    fill_method = 'ffill'
+                binner = (pd.Series(binner, index=binner)
+                          .reindex(times, method=fill_method))
+                bin_actual = np.unique(binner.values)
+                label_dict = dict(zip(bin_actual, labels.values))
+                # np.unique returns --sorted-- unique values
+                binner = binner.map(label_dict)
+                grouper = ('downsampling', pd.Index(labels), binner)
+            else:
+                # if we're upsampling CFTimeIndex, do this:
+                binner = _adjust_binner_for_upsample(binner, closed)
+                grouper = ('upsampling', pd.Index(labels), binner, closed)
+        else:
+            grouper = pd.Grouper(freq=freq, closed=closed, label=label, base=base)
 
         group = DataArray(dim, [(dim.dims, dim)], name=RESAMPLE_DIM)
-        grouper = pd.Grouper(freq=freq, closed=closed, label=label, base=base)
         resampler = self._resample_cls(self, group=group, dim=dim_name,
                                        grouper=grouper,
                                        resample_dim=RESAMPLE_DIM)

xarray/core/dataarray.py

@@ -16,7 +16,7 @@
     assert_coordinate_consistent, remap_label_indexers)
 from .dataset import Dataset, merge_indexes, split_indexes
 from .formatting import format_item
-from .options import OPTIONS, _get_keep_attrs
+from .options import OPTIONS
 from .pycompat import OrderedDict, basestring, iteritems, range, zip
 from .utils import (
     decode_numpy_dict_values, either_dict_or_kwargs, ensure_us_time_resolution)

xarray/core/groupby.py

@@ -234,7 +234,29 @@ def __init__(self, obj, group, squeeze=False, grouper=None, bins=None,
                 # TODO: sort instead of raising an error
                 raise ValueError('index must be monotonic for resampling')
             s = pd.Series(np.arange(index.size), index)
-            first_items = s.groupby(grouper).first()
+            if isinstance(grouper, tuple):
+                if grouper[0] == 'downsampling':
+                    # if we're downsampling CFTimeIndex, do this:
+                    labels = grouper[1]
+                    binner = grouper[2]
+                    first_items = s.groupby(binner).first().reindex(labels)
+                    # reindex(grouper[1]) adds empty np.nan bins to
+                    # emulate pandas behavior
+                elif grouper[0] == 'upsampling':
+                    # if we're upsampling CFTimeIndex, do this:
+                    labels = grouper[1]
+                    binner = grouper[2]
+                    closed = grouper[3]
+                    if closed == 'right':
+                        first_items = s.reindex(pd.Index(binner),
+                                                method='nearest')
+                        first_items.index = labels
+                    else:
+                        first_items = s.reindex(pd.Index(binner),
+                                                method='bfill')
+                        first_items.index = labels
+            else:
+                first_items = s.groupby(grouper).first()
             full_index = first_items.index
             if first_items.isnull().any():
                 first_items = first_items.dropna()

xarray/core/options.py

@@ -63,7 +63,8 @@ def _get_keep_attrs(default):
     elif global_choice in [True, False]:
         return global_choice
     else:
-        raise ValueError("The global option keep_attrs must be one of True, False or 'default'.")
+        raise ValueError("The global option keep_attrs must be one of"
+                         " True, False or 'default'.")
 
 
 class set_options(object):

xarray/core/resample.py

@@ -194,14 +194,35 @@ def _interpolate(self, kind='linear'):
                 .format(self._obj.data.name)
             )
 
-        x = self._obj[self._dim].astype('float')
+        # from ..coding.cftimeindex import CFTimeIndex
+        import cftime as cf
+        import numpy as np
+        if isinstance(self._obj[self._dim].values[0], cf.datetime):
+            t = self._obj[self._dim]
+            x = np.insert([td.total_seconds() for td in
+                           t[1:].values - t[:-1].values], 0, 0).cumsum()
+            #  calling total_seconds is potentially bad for performance
+            x = x.round()
+            # Rounding fixes erroneous microsecond offsets in timedelta
+            # (fault of CFTime), but destroys microsecond resolution data
+        else:
+            x = self._obj[self._dim].astype('float')
         y = self._obj.data
 
         axis = self._obj.get_axis_num(self._dim)
 
         f = interp1d(x, y, kind=kind, axis=axis, bounds_error=True,
                      assume_sorted=True)
-        new_x = self._full_index.values.astype('float')
+        if isinstance(self._full_index.values[0], cf.datetime):
+            t = self._full_index
+            new_x = np.insert([td.total_seconds() for td in
+                               t[1:].values - t[:-1].values], 0, 0).cumsum()
+            #  calling total_seconds is potentially bad for performance
+            new_x = new_x.round()
+            # Rounding fixes erroneous microsecond offsets in timedelta
+            # (fault of CFTime), but destroys microsecond resolution data
+        else:
+            new_x = self._full_index.values.astype('float')
 
         # construct new up-sampled DataArray
         dummy = self._obj.copy()

xarray/core/resample_cftime.py

@@ -0,0 +1,151 @@
+"""
+CFTimeIndex port of pandas resampling
+(pandas/pandas/core/resample.py)
+Does not support non-integer freq
+"""
+from __future__ import absolute_import, division, print_function
+
+import datetime
+from ..coding.cftimeindex import CFTimeIndex
+from ..coding.cftime_offsets import (cftime_range, normalize_date,
+                                     Day, Hour, Minute, Second)
+
+
+def _get_time_bins(index, freq, closed, label, base):
+    # This portion of code comes from TimeGrouper __init__ #
+    end_types = {'M', 'A'}
+    if freq._freq in end_types:
+        if closed is None:
+            closed = 'right'
+        if label is None:
+            label = 'right'
+    else:
+        if closed is None:
+            closed = 'left'
+        if label is None:
+            label = 'left'
+    # This portion of code comes from TimeGrouper __init__ #
+
+    if not isinstance(index, CFTimeIndex):
+        raise TypeError('index must be a CFTimeIndex, but got '
+                        'an instance of %r' % type(index).__name__)
+    if len(index) == 0:
+        binner = labels = CFTimeIndex(data=[], name=index.name)
+        return binner, [], labels
+
+    first, last = _get_range_edges(index.min(), index.max(), freq,
+                                   closed=closed,
+                                   base=base)
+    binner = labels = cftime_range(freq=freq,
+                                   start=first,
+                                   end=last,
+                                   name=index.name)
+
+    if len(binner) > 1 and binner[-1] < last:
+        extra_date_range = cftime_range(binner[-1], last + freq,
+                                        freq=freq, name=index.name)
+        binner = labels = CFTimeIndex(binner.append(extra_date_range[1:]))
+
+    trimmed = False
+    if len(binner) > 2 and binner[-2] == last and closed == 'right':
+        binner = binner[:-1]
+        trimmed = True
+
+    if closed == 'right':
+        labels = binner
+        if label == 'right':
+            labels = labels[1:]
+        elif not trimmed:
+            labels = labels[:-1]
+    else:
+        if label == 'right':
+            labels = labels[1:]
+        elif not trimmed:
+            labels = labels[:-1]
+    return binner, labels
+
+
+def _adjust_bin_edges(binner, ax_values, freq):
+    # Some hacks for > daily data, see #1471, #1458, #1483
+    if freq._freq not in ['D', 'H', 'T', 'min', 'S']:
+        # intraday values on last day
+        if binner[-2] > ax_values.max():
+            binner = binner[:-1]
+    return binner
+
+
+def _get_range_edges(first, last, offset, closed='left', base=0):
+    if offset._freq in ['D', 'H', 'T', 'min', 'S']:
+        is_day = isinstance(offset, Day)
+        if (is_day and offset.n == 1) or not is_day:
+            return _adjust_dates_anchored(first, last, offset,
+                                          closed=closed, base=base)
+    else:
+        first = normalize_date(first)
+        last = normalize_date(last)
+
+    if closed == 'left':
+        first = offset.rollback(first)
+    else:
+        first = first - offset
+
+    last = last + offset
+    return first, last
+
+
+def _adjust_dates_anchored(first, last, offset, closed='right', base=0):
+    base = base % offset.n
+    start_day = normalize_date(first)
+    base_td = datetime.timedelta(0)
+    if offset._freq == 'D':
+        base_td = datetime.timedelta(days=base)
+    elif offset._freq == 'H':
+        base_td = datetime.timedelta(hours=base)
+    elif offset._freq in ['T', 'min']:
+        base_td = datetime.timedelta(minutes=base)
+    elif offset._freq == 'S':
+        base_td = datetime.timedelta(seconds=base)
+    offset_td = _offset_timedelta(offset)
+    start_day += base_td
+    foffset = (first - start_day) % offset_td
+    loffset = (last - start_day) % offset_td
+    if closed == 'right':
+        if foffset.total_seconds() > 0:
+            fresult = first - foffset
+        else:
+            fresult = first - offset_td
+
+        if loffset.total_seconds() > 0:
+            lresult = last + (offset_td - loffset)
+        else:
+            lresult = last
+    else:
+        if foffset.total_seconds() > 0:
+            fresult = first - foffset
+        else:
+            fresult = first
+
+        if loffset.total_seconds() > 0:
+            lresult = last + (offset_td - loffset)
+        else:
+            lresult = last + offset_td
+    return fresult, lresult
+
+
+def _offset_timedelta(offset):
+    if isinstance(offset, Day):
+        return datetime.timedelta(days=offset.n)
+    elif isinstance(offset, Hour):
+        return datetime.timedelta(hours=offset.n)
+    elif isinstance(offset, Minute):
+        return datetime.timedelta(minutes=offset.n)
+    elif isinstance(offset, Second):
+        return datetime.timedelta(seconds=offset.n)
+
+
+def _adjust_binner_for_upsample(binner, closed):
+    if closed == 'right':
+        binner = binner[1:]
+    else:
+        binner = binner[:-1]
+    return binner

xarray/plot/plot.py

@@ -14,7 +14,7 @@
 import numpy as np
 import pandas as pd
 
-from xarray.core.alignment import align
+# from xarray.core.alignment import align
 from xarray.core.common import contains_cftime_datetimes
 from xarray.core.pycompat import basestring
 
@@ -255,7 +255,6 @@ def _infer_line_data(darray, x, y, hue):
         huelabel = label_from_attrs(darray[huename])
         hueplt = darray[huename]
 
-
     xlabel = label_from_attrs(xplt)
     ylabel = label_from_attrs(yplt)