BUG: Series.combine() fails with ExtensionArray inside of Series

doc/source/whatsnew/v0.24.0.txt

@@ -174,6 +174,7 @@ Reshaping
 Other
 ^^^^^
 
+- :meth:`Series.combine()` no longer fails with :class:`~pandas.api.extensions.ExtensionArray` inside of :class:`Series` (:issue:`20825`)
 -
 -
 -

pandas/core/series.py

@@ -2185,18 +2185,33 @@ def _binop(self, other, func, level=None, fill_value=None):
 
         this_vals, other_vals = ops.fill_binop(this.values, other.values,
                                                fill_value)
-
-        with np.errstate(all='ignore'):
-            result = func(this_vals, other_vals)
         name = ops.get_op_result_name(self, other)
+
+        if (is_extension_array_dtype(this_vals) or
+                is_extension_array_dtype(other_vals)):
+            try:
+                result = func(this_vals, other_vals)
+            except TypeError:
+                result = NotImplemented
+
+            if result is NotImplemented:
+                result = [func(a, b) for a, b in zip(this_vals, other_vals)]
+                if is_extension_array_dtype(this_vals):
+                    excons = type(this_vals)._from_sequence
+                else:
+                    excons = type(other_vals)._from_sequence
+                result = excons(result)
+        else:
+            with np.errstate(all='ignore'):
+                result = func(this_vals, other_vals)
         result = self._constructor(result, index=new_index, name=name)
         result = result.__finalize__(self)
         if name is None:
             # When name is None, __finalize__ overwrites current name
             result.name = None
         return result
 
-    def combine(self, other, func, fill_value=np.nan):
+    def combine(self, other, func, fill_value=None):
         """
         Perform elementwise binary operation on two Series using given function
         with optional fill value when an index is missing from one Series or
@@ -2208,6 +2223,9 @@ def combine(self, other, func, fill_value=np.nan):
         func : function
             Function that takes two scalars as inputs and return a scalar
         fill_value : scalar value
+            The default specifies to use np.nan unless self is
+            backed by ExtensionArray, in which case the ExtensionArray
+            na_value is used.
 
         Returns
         -------
@@ -2227,20 +2245,36 @@ def combine(self, other, func, fill_value=np.nan):
         Series.combine_first : Combine Series values, choosing the calling
             Series's values first
         """
+        self_is_ext = is_extension_array_dtype(self.values)
+        if fill_value is None:
+            if self_is_ext:
+                fill_value = self.dtype.na_value
+            else:
+                fill_value = np.nan
         if isinstance(other, Series):
             new_index = self.index.union(other.index)
             new_name = ops.get_op_result_name(self, other)
-            new_values = np.empty(len(new_index), dtype=self.dtype)
-            for i, idx in enumerate(new_index):
+            new_values = []
+            for idx in new_index:
                 lv = self.get(idx, fill_value)
                 rv = other.get(idx, fill_value)
                 with np.errstate(all='ignore'):
-                    new_values[i] = func(lv, rv)
+                    new_values.append(func(lv, rv))
         else:
             new_index = self.index
-            with np.errstate(all='ignore'):
-                new_values = func(self._values, other)
+            if not self_is_ext:
+                with np.errstate(all='ignore'):
+                    new_values = func(self._values, other)
+            else:
+                new_values = [func(lv, other) for lv in self._values]
             new_name = self.name
+
+        if self_is_ext and not is_categorical_dtype(self.values):
+            try:
+                new_values = self._values._from_sequence(new_values)
+            except TypeError:
+                pass
+
         return self._constructor(new_values, index=new_index, name=new_name)
 
     def combine_first(self, other):

pandas/tests/extension/category/test_categorical.py

@@ -2,6 +2,9 @@
 
 import pytest
 import numpy as np
+import pandas as pd
+
+import pandas.util.testing as tm
 
 from pandas.api.types import CategoricalDtype
 from pandas import Categorical
@@ -154,6 +157,17 @@ class TestMethods(base.BaseMethodsTests):
     def test_value_counts(self, all_data, dropna):
         pass
 
+    def test_combine(self):
+        # GH 20825
+        orig_data1 = make_data()
+        orig_data2 = make_data()
+        s1 = pd.Series(Categorical(orig_data1, ordered=True))
+        s2 = pd.Series(Categorical(orig_data2, ordered=True))
+        result = s1.combine(s2, lambda x1, x2: x1 <= x2)
+        expected = pd.Series([a <= b for (a, b) in
+                              zip(orig_data1, orig_data2)])
+        tm.assert_series_equal(result, expected)
+
 
 class TestCasting(base.BaseCastingTests):
     pass

pandas/tests/extension/decimal/array.py

@@ -28,7 +28,9 @@ class DecimalArray(ExtensionArray):
     dtype = DecimalDtype()
 
     def __init__(self, values):
-        assert all(isinstance(v, decimal.Decimal) for v in values)
+        for val in values:
+            if not isinstance(val, self.dtype.type):
+                raise TypeError
         values = np.asarray(values, dtype=object)
 
         self._data = values

pandas/tests/extension/decimal/test_decimal.py

@@ -138,6 +138,22 @@ def test_value_counts(self, all_data, dropna):
 
         tm.assert_series_equal(result, expected)
 
+    def test_combine(self):
+        # GH 20825
+        orig_data1 = make_data()
+        orig_data2 = make_data()
+        s1 = pd.Series(DecimalArray(orig_data1))
+        s2 = pd.Series(DecimalArray(orig_data2))
+        result = s1.combine(s2, lambda x1, x2: x1 <= x2)
+        expected = pd.Series([a <= b for (a, b) in
+                              zip(orig_data1, orig_data2)])
+        tm.assert_series_equal(result, expected)
+
+        result = s1.combine(s2, lambda x1, x2: x1 + x2)
+        expected = pd.Series(DecimalArray([a + b for (a, b) in
+                                           zip(orig_data1, orig_data2)]))
+        tm.assert_series_equal(result, expected)
+
 
 class TestCasting(BaseDecimal, base.BaseCastingTests):
     pass