ENH - Index set operation modifications to address issue #23525

pandas/core/indexes/base.py

@@ -2756,6 +2756,25 @@ def _get_reconciled_name_object(self, other):
             return self._shallow_copy(name=name)
         return self
 
+    def _union_incompatible_dtypes(self, other):
+        """
+        Casts this and other index to object dtype to allow the formation
+        of a union between incompatible types.
+        """
+        this = self.astype('O')
+        # call Index for when `other` is list-like
+        other = Index(other).astype('O')
+        return Index.union(this, other).astype('O')
+
+    def _is_compatible_with_other(self, other):
+        """
+        Check whether this and the other dtype are compatible with each other.
+        Meaning a union can be formed between them without needing to be cast
+        to dtype object.
+        """
+        return (type(self) is type(other) 
+                and is_dtype_equal(self.dtype, other.dtype))
+
     def union(self, other):
         """
         Form the union of two Index objects and sorts if possible.
@@ -2778,22 +2797,28 @@ def union(self, other):
 
         """
         self._assert_can_do_setop(other)
+
+        if not self._is_compatible_with_other(other):
+            return self._union_incompatible_dtypes(other)
+
+        # This line needs to be after _union_incompatible_dtypes to ensure 
+        # the original type of other is not lost after being cast to Index
         other = ensure_index(other)
+        return self._union(other)
 
-        if len(other) == 0 or self.equals(other):
-            return self._get_reconciled_name_object(other)
+    def _union(self, other):
+        """
+        Specific union logic should go here. In subclasses union behavior
+        should be overwritten here rather than in `self.union`
+        """
 
         if len(self) == 0:
             return other._get_reconciled_name_object(self)
+        elif len(other) == 0:
+            return self._get_reconciled_name_object(other)
 
-        # TODO: is_dtype_union_equal is a hack around
-        # 1. buggy set ops with duplicates (GH #13432)
-        # 2. CategoricalIndex lacking setops (GH #10186)
-        # Once those are fixed, this workaround can be removed
-        if not is_dtype_union_equal(self.dtype, other.dtype):
-            this = self.astype('O')
-            other = other.astype('O')
-            return this.union(other)
+        if self.equals(other):
+            return self._get_reconciled_name_object(other)
 
         # TODO(EA): setops-refactor, clean all this up
         if is_period_dtype(self) or is_datetime64tz_dtype(self):

pandas/core/indexes/category.py

@@ -872,6 +872,9 @@ def _delegate_method(self, name, *args, **kwargs):
             return res
         return CategoricalIndex(res, name=self.name)
 
+    def _is_compatible_with_other(self, other):
+        return type(self) is type(other) and type(self.dtype) is type(self.dtype)
+
 
 CategoricalIndex._add_numeric_methods_add_sub_disabled()
 CategoricalIndex._add_numeric_methods_disabled()

pandas/core/indexes/datetimes.py

@@ -562,6 +562,13 @@ def unique(self, level=None):
         result = super(DatetimeIndex, naive).unique(level=level)
         return self._shallow_copy(result.values)
 
+    def _is_compatible_with_other(self, other):
+        is_compat = super(DatetimeIndex, self)._is_compatible_with_other(other)
+        if not is_compat:
+            is_compat = (hasattr(other, 'dtype') 
+                         and self.dtype.base == other.dtype.base)
+        return is_compat
+
     def union(self, other):
         """
         Specialized union for DatetimeIndex objects. If combine
@@ -576,10 +583,11 @@ def union(self, other):
         -------
         y : Index or DatetimeIndex
         """
-        self._assert_can_do_setop(other)
+        return super(DatetimeIndex, self).union(other)
 
+    def _union(self, other):
         if len(other) == 0 or self.equals(other) or len(self) == 0:
-            return super(DatetimeIndex, self).union(other)
+            return super(DatetimeIndex, self)._union(other)
 
         if not isinstance(other, DatetimeIndex):
             try:
@@ -592,7 +600,7 @@ def union(self, other):
         if this._can_fast_union(other):
             return this._fast_union(other)
         else:
-            result = Index.union(this, other)
+            result = Index._union(this, other)
             if isinstance(result, DatetimeIndex):
                 result._tz = timezones.tz_standardize(this.tz)
                 if (result.freq is None and

pandas/core/indexes/interval.py

@@ -1038,7 +1038,16 @@ def overlaps(self, other):
 
     def _setop(op_name):
         def func(self, other):
-            other = self._as_like_interval_index(other)
+            try:
+                other = self._as_like_interval_index(other)
+            # allow ValueError from this method to raise to catch mixed closed
+            # except only Non-Interval index mismatches.
+            except TypeError:
+                # Currently this will cause difference operations to return
+                # object dtype as opposed to IntervalIndex, unlike other Index
+                # objects that return the same type when using `difference` on
+                # mismatched types
+                return getattr(self.astype('O'), op_name)(other)
 
             # GH 19016: ensure set op will not return a prohibited dtype
             subtypes = [self.dtype.subtype, other.dtype.subtype]
@@ -1059,6 +1068,7 @@ def func(self, other):
 
             return type(self).from_tuples(result, closed=self.closed,
                                           name=result_name)
+
         return func
 
     @property

pandas/core/indexes/numeric.py

@@ -228,6 +228,15 @@ def _assert_safe_casting(cls, data, subarr):
                 raise TypeError('Unsafe NumPy casting, you must '
                                 'explicitly cast')
 
+    def _is_compatible_with_other(self, other):
+        from pandas.core.dtypes.generic import ABCRangeIndex
+        is_compat = super(Int64Index, self)._is_compatible_with_other(other)
+        if not is_compat:
+            is_compat = (type(self) is Int64Index 
+                         and isinstance(other, ABCRangeIndex))
+        return is_compat
+
+
 
 Int64Index._add_numeric_methods()
 Int64Index._add_logical_methods()

pandas/core/indexes/period.py

@@ -833,6 +833,11 @@ def join(self, other, how='left', level=None, return_indexers=False,
         """
         self._assert_can_do_setop(other)
 
+        if not isinstance(other, PeriodIndex):
+            return self.astype('O').join(other, how=how, level=level,
+                                         return_indexers=return_indexers,
+                                         sort=sort)
+
         result = Int64Index.join(self, other, how=how, level=level,
                                  return_indexers=return_indexers,
                                  sort=sort)
@@ -845,10 +850,9 @@ def join(self, other, how='left', level=None, return_indexers=False,
     def _assert_can_do_setop(self, other):
         super(PeriodIndex, self)._assert_can_do_setop(other)
 
-        if not isinstance(other, PeriodIndex):
-            raise ValueError('can only call with other PeriodIndex-ed objects')
-
-        if self.freq != other.freq:
+        # *Can't* use PeriodIndexes of different freqs
+        # *Can* use PeriodIndex/DatetimeIndex
+        if isinstance(other, PeriodIndex) and self.freq != other.freq:
             msg = DIFFERENT_FREQ_INDEX.format(self.freqstr, other.freqstr)
             raise IncompatibleFrequency(msg)
 

pandas/core/indexes/range.py

@@ -416,6 +416,13 @@ def _extended_gcd(self, a, b):
             old_t, t = t, old_t - quotient * t
         return old_r, old_s, old_t
 
+
+    def _is_compatible_with_other(self, other):
+        is_compat = super(RangeIndex, self)._is_compatible_with_other(other)
+        if not is_compat:
+            is_compat = type(other) is Int64Index
+        return is_compat
+
     def union(self, other):
         """
         Form the union of two Index objects and sorts if possible
@@ -428,9 +435,11 @@ def union(self, other):
         -------
         union : Index
         """
-        self._assert_can_do_setop(other)
+        return super(RangeIndex, self).union(other)
+
+    def _union(self, other):
         if len(other) == 0 or self.equals(other) or len(self) == 0:
-            return super(RangeIndex, self).union(other)
+            return super(RangeIndex, self)._union(other)
 
         if isinstance(other, RangeIndex):
             start_s, step_s = self._start, self._step
@@ -469,7 +478,7 @@ def union(self, other):
                         (end_s - step_o <= end_o)):
                     return RangeIndex(start_r, end_r + step_o, step_o)
 
-        return self._int64index.union(other)
+        return self._int64index._union(other)
 
     @Appender(_index_shared_docs['join'])
     def join(self, other, how='left', level=None, return_indexers=False,

pandas/core/indexes/timedeltas.py

@@ -263,10 +263,11 @@ def union(self, other):
         -------
         y : Index or TimedeltaIndex
         """
-        self._assert_can_do_setop(other)
+        return super(TimedeltaIndex, self).union(other)
 
+    def _union(self, other):
         if len(other) == 0 or self.equals(other) or len(self) == 0:
-            return super(TimedeltaIndex, self).union(other)
+            return super(TimedeltaIndex, self)._union(other)
 
         if not isinstance(other, TimedeltaIndex):
             try:
@@ -278,7 +279,7 @@ def union(self, other):
         if this._can_fast_union(other):
             return this._fast_union(other)
         else:
-            result = Index.union(this, other)
+            result = Index._union(this, other)
             if isinstance(result, TimedeltaIndex):
                 if result.freq is None:
                     result.freq = to_offset(result.inferred_freq)

pandas/tests/indexes/common.py

@@ -624,11 +624,7 @@ def test_intersection_base(self):
             cases = [klass(second.values)
                      for klass in [np.array, Series, list]]
             for case in cases:
-                if isinstance(idx, PeriodIndex):
-                    msg = "can only call with other PeriodIndex-ed objects"
-                    with pytest.raises(ValueError, match=msg):
-                        first.intersection(case)
-                elif isinstance(idx, CategoricalIndex):
+                if isinstance(idx, CategoricalIndex):
                     pass
                 else:
                     result = first.intersection(case)
@@ -651,11 +647,7 @@ def test_union_base(self):
             cases = [klass(second.values)
                      for klass in [np.array, Series, list]]
             for case in cases:
-                if isinstance(idx, PeriodIndex):
-                    msg = "can only call with other PeriodIndex-ed objects"
-                    with pytest.raises(ValueError, match=msg):
-                        first.union(case)
-                elif isinstance(idx, CategoricalIndex):
+                if isinstance(idx, CategoricalIndex):
                     pass
                 else:
                     result = first.union(case)
@@ -682,11 +674,7 @@ def test_difference_base(self):
             cases = [klass(second.values)
                      for klass in [np.array, Series, list]]
             for case in cases:
-                if isinstance(idx, PeriodIndex):
-                    msg = "can only call with other PeriodIndex-ed objects"
-                    with pytest.raises(ValueError, match=msg):
-                        first.difference(case)
-                elif isinstance(idx, CategoricalIndex):
+                if isinstance(idx, CategoricalIndex):
                     pass
                 elif isinstance(idx, (DatetimeIndex, TimedeltaIndex)):
                     assert result.__class__ == answer.__class__
@@ -716,11 +704,7 @@ def test_symmetric_difference(self):
             cases = [klass(second.values)
                      for klass in [np.array, Series, list]]
             for case in cases:
-                if isinstance(idx, PeriodIndex):
-                    msg = "can only call with other PeriodIndex-ed objects"
-                    with pytest.raises(ValueError, match=msg):
-                        first.symmetric_difference(case)
-                elif isinstance(idx, CategoricalIndex):
+                if isinstance(idx, CategoricalIndex):
                     pass
                 else:
                     result = first.symmetric_difference(case)

pandas/tests/indexes/conftest.py

@@ -8,6 +8,7 @@
 import pandas.util.testing as tm
 
 
+# add inteval index?
 @pytest.fixture(params=[tm.makeUnicodeIndex(100),
                         tm.makeStringIndex(100),
                         tm.makeDateIndex(100),
@@ -28,6 +29,31 @@ def indices(request):
     return request.param
 
 
+def _make_repeating_index(x=10):
+    # x should be > 1
+    return Index(sorted([i for i in range(x//2 + 1)] * 2)[:x])
+
+
+@pytest.fixture(params=[tm.makeUnicodeIndex,
+                        tm.makeStringIndex,
+                        tm.makeDateIndex,
+                        tm.makePeriodIndex,
+                        tm.makeTimedeltaIndex,
+                        tm.makeIntIndex,
+                        tm.makeUIntIndex,
+                        tm.makeRangeIndex,
+                        tm.makeFloatIndex,
+                        lambda x=10: Index(np.random.choice([True, False], x)),
+                        tm.makeCategoricalIndex,
+                        lambda x=None: Index([]),
+                        tm.makeMultiIndex,
+                        _make_repeating_index,
+                        tm.makeIntervalIndex],
+                ids=lambda x: type(x).__name__)
+def index_factory(request):
+    return request.param
+
+
 @pytest.fixture(params=[1, np.array(1, dtype=np.int64)])
 def one(request):
     # zero-dim integer array behaves like an integer

pandas/tests/indexes/datetimes/test_datetime.py

@@ -303,9 +303,9 @@ def test_join_with_period_index(self, join_type):
             c_idx_type='p', r_idx_type='dt')
         s = df.iloc[:5, 0]
 
-        msg = 'can only call with other PeriodIndex-ed objects'
-        with pytest.raises(ValueError, match=msg):
-            df.columns.join(s.index, how=join_type)
+        expected = df.columns.astype('O').join(s.index, how=join_type)
+        result = df.columns.join(s.index, how=join_type)
+        tm.assert_index_equal(expected, result)
 
     def test_factorize(self):
         idx1 = DatetimeIndex(['2014-01', '2014-01', '2014-02', '2014-02',

pandas/tests/indexes/datetimes/test_setops.py

@@ -29,10 +29,13 @@ def test_union2(self):
         assert tm.equalContents(union, everything)
 
         # GH 10149
+        expected = first.astype('O').union(
+            pd.Index(second.values, dtype='O')
+        ).astype('O')
         cases = [klass(second.values) for klass in [np.array, Series, list]]
         for case in cases:
             result = first.union(case)
-            assert tm.equalContents(result, everything)
+            assert tm.equalContents(result, expected)
 
     @pytest.mark.parametrize("tz", tz)
     def test_union(self, tz):
@@ -256,11 +259,11 @@ def test_datetimeindex_union_join_empty(self):
         empty = Index([])
 
         result = dti.union(empty)
-        assert isinstance(result, DatetimeIndex)
-        assert result is result
+        tm.assert_index_equal(result, dti.astype('O'))
 
         result = dti.join(empty)
         assert isinstance(result, DatetimeIndex)
+        tm.assert_index_equal(result, dti)
 
     def test_join_nonunique(self):
         idx1 = to_datetime(['2012-11-06 16:00:11.477563',

pandas/tests/indexes/interval/test_interval.py

@@ -835,15 +835,16 @@ def test_symmetric_difference(self, closed):
 
     @pytest.mark.parametrize('op_name', [
         'union', 'intersection', 'difference', 'symmetric_difference'])
-    def test_set_operation_errors(self, closed, op_name):
+    def test_set_incompatible_types(self, closed, op_name):
         index = self.create_index(closed=closed)
         set_op = getattr(index, op_name)
 
-        # non-IntervalIndex
-        msg = ('the other index needs to be an IntervalIndex too, but '
-               'was type Int64Index')
-        with pytest.raises(TypeError, match=msg):
-            set_op(Index([1, 2, 3]))
+        # non-IntervalIndexf
+        expected = getattr(index.astype('O'), op_name)(Index([1, 2, 3]))
+        result = set_op(Index([1, 2, 3]))
+        tm.assert_index_equal(result, expected)
+
+        # Come back to mixed interval types
 
         # mixed closed
         msg = ('can only do set operations between two IntervalIndex objects '