Pandas version checks
-
[X] I have checked that this issue has not already been reported.
-
[X] I have confirmed this bug exists on the latest version of pandas.
-
[ ] I have confirmed this bug exists on the main branch of pandas.
Reproducible Example
import pandas as pd
s1 = pd.Series(
[23, 22, 21], index=pd.Index(["a", "b", "c"], name="index a"), dtype=pd.Int64Dtype()
)
s2 = pd.Series(
[21, 22, 23],
index=pd.Index(["a", "b", "c"], name="index b", dtype=pd.StringDtype()),
dtype=pd.Int64Dtype(),
)
s1 - s2
Issue Description
This example with two series with a nullable integer data type and a index with different names and of differing data types, on with a nullable string datatype and object datatype fails with and error from pandas._libs.algos.ensure_platform_int()
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
Cell In[170], line 10
1 s1 = pd.Series(
2 [23, 22, 21], index=pd.Index(["a", "b", "c"], name="index a"), dtype=pd.Int64Dtype()
3 )
4 s2 = pd.Series(
5 [21, 22, 23],
6 index=pd.Index(["a", "b", "c"], name="index b", dtype=pd.StringDtype()),
7 dtype=pd.Int64Dtype(),
8 )
---> 10 s1 - s2
File ~/beftett/.venv/lib/python3.10/site-packages/pandas/core/ops/common.py:76, in _unpack_zerodim_and_defer.<locals>.new_method(self, other)
72 return NotImplemented
74 other = item_from_zerodim(other)
---> 76 return method(self, other)
File ~/beftett/.venv/lib/python3.10/site-packages/pandas/core/arraylike.py:194, in OpsMixin.__sub__(self, other)
192 @unpack_zerodim_and_defer("__sub__")
193 def __sub__(self, other):
--> 194 return self._arith_method(other, operator.sub)
File ~/beftett/.venv/lib/python3.10/site-packages/pandas/core/series.py:5814, in Series._arith_method(self, other, op)
5813 def _arith_method(self, other, op):
-> 5814 self, other = self._align_for_op(other)
5815 return base.IndexOpsMixin._arith_method(self, other, op)
File ~/beftett/.venv/lib/python3.10/site-packages/pandas/core/series.py:5844, in Series._align_for_op(self, right, align_asobject)
5841 left = left.astype(object)
5842 right = right.astype(object)
-> 5844 left, right = left.align(right, copy=False)
5846 return left, right
File ~/beftett/.venv/lib/python3.10/site-packages/pandas/core/generic.py:10095, in NDFrame.align(self, other, join, axis, level, copy, fill_value, method, limit, fill_axis, broadcast_axis)
10082 left, _right, join_index = self._align_frame(
10083 other,
10084 join=join,
(...)
10091 fill_axis=fill_axis,
10092 )
10094 elif isinstance(other, ABCSeries):
> 10095 left, _right, join_index = self._align_series(
10096 other,
10097 join=join,
10098 axis=axis,
10099 level=level,
10100 copy=copy,
10101 fill_value=fill_value,
10102 method=method,
10103 limit=limit,
10104 fill_axis=fill_axis,
10105 )
10106 else: # pragma: no cover
10107 raise TypeError(f"unsupported type: {type(other)}")
File ~/beftett/.venv/lib/python3.10/site-packages/pandas/core/generic.py:10224, in NDFrame._align_series(self, other, join, axis, level, copy, fill_value, method, limit, fill_axis)
10221 new_mgr = self._mgr.reindex_indexer(join_index, lidx, axis=1, copy=copy)
10222 left = self._constructor_from_mgr(new_mgr, axes=new_mgr.axes)
> 10224 right = other._reindex_indexer(join_index, ridx, copy)
10226 else:
10227 # one has > 1 ndim
10228 fdata = self._mgr
File ~/beftett/.venv/lib/python3.10/site-packages/pandas/core/series.py:4779, in Series._reindex_indexer(self, new_index, indexer, copy)
4776 return self.copy(deep=copy)
4777 return self
-> 4779 new_values = algorithms.take_nd(
4780 self._values, indexer, allow_fill=True, fill_value=None
4781 )
4782 return self._constructor(new_values, index=new_index, copy=False)
File ~/beftett/.venv/lib/python3.10/site-packages/pandas/core/array_algos/take.py:115, in take_nd(arr, indexer, axis, fill_value, allow_fill)
110 arr = cast("NDArrayBackedExtensionArray", arr)
111 return arr.take(
112 indexer, fill_value=fill_value, allow_fill=allow_fill, axis=axis
113 )
--> 115 return arr.take(indexer, fill_value=fill_value, allow_fill=allow_fill)
117 arr = np.asarray(arr)
118 return _take_nd_ndarray(arr, indexer, axis, fill_value, allow_fill)
File ~/beftett/.venv/lib/python3.10/site-packages/pandas/core/arrays/masked.py:905, in BaseMaskedArray.take(self, indexer, allow_fill, fill_value, axis)
894 def take(
895 self,
896 indexer,
(...)
902 # we always fill with 1 internally
903 # to avoid upcasting
904 data_fill_value = self._internal_fill_value if isna(fill_value) else fill_value
--> 905 result = take(
906 self._data,
907 indexer,
908 fill_value=data_fill_value,
909 allow_fill=allow_fill,
910 axis=axis,
911 )
913 mask = take(
914 self._mask, indexer, fill_value=True, allow_fill=allow_fill, axis=axis
915 )
917 # if we are filling
918 # we only fill where the indexer is null
919 # not existing missing values
920 # TODO(jreback) what if we have a non-na float as a fill value?
File ~/beftett/.venv/lib/python3.10/site-packages/pandas/core/algorithms.py:1309, in take(arr, indices, axis, allow_fill, fill_value)
1306 if not is_array_like(arr):
1307 arr = np.asarray(arr)
-> 1309 indices = ensure_platform_int(indices)
1311 if allow_fill:
1312 # Pandas style, -1 means NA
1313 validate_indices(indices, arr.shape[axis])
File pandas/_libs/algos_common_helper.pxi:22, in pandas._libs.algos.ensure_platform_int()
TypeError: int() argument must be a string, a bytes-like object or a real number, not 'NoneType'
If any of these three conditions is removed, the subtraction succeeds.
Expected Behavior
The subtraction should succeed.
Installed Versions
Comment From: rhshadrach
Thanks for the report. Might be related to #45564 - cc @phofl.
https://github.com/pandas-dev/pandas/blob/95308514e1221200e4526dfaf248283f3d7ade06/pandas/core/series.py#L4637-L4646
Prior to this PR, we would not call take_nd when indexer is None. Now we might be passing indexer=None to take_nd. For NumPy arrays, we end up in _take_nd_array which has the lines:
https://github.com/pandas-dev/pandas/blob/95308514e1221200e4526dfaf248283f3d7ade06/pandas/core/array_algos/take.py#L127-L129
but no such logic exists for EAs. Perhaps it should?
Comment From: Adasjo
take
Comment From: CasperKristiansson
take
Comment From: VictorOsterman
take
Comment From: Tobiky
take
Comment From: Lindefor
take
Comment From: jbrockmendel
works on main. could use a test (or check if one already exists)
Comment From: lesliejk
Would like to work on tests or check coverage for this issue.