"""
|
missing types & inference
|
"""
|
from __future__ import annotations
|
|
from decimal import Decimal
|
from functools import partial
|
from typing import (
|
TYPE_CHECKING,
|
overload,
|
)
|
|
import numpy as np
|
|
from pandas._config import get_option
|
|
from pandas._libs import lib
|
import pandas._libs.missing as libmissing
|
from pandas._libs.tslibs import (
|
NaT,
|
iNaT,
|
)
|
|
from pandas.core.dtypes.common import (
|
DT64NS_DTYPE,
|
TD64NS_DTYPE,
|
ensure_object,
|
is_bool_dtype,
|
is_categorical_dtype,
|
is_complex_dtype,
|
is_dtype_equal,
|
is_extension_array_dtype,
|
is_float_dtype,
|
is_integer_dtype,
|
is_object_dtype,
|
is_scalar,
|
is_string_or_object_np_dtype,
|
needs_i8_conversion,
|
)
|
from pandas.core.dtypes.dtypes import (
|
CategoricalDtype,
|
DatetimeTZDtype,
|
ExtensionDtype,
|
IntervalDtype,
|
PeriodDtype,
|
)
|
from pandas.core.dtypes.generic import (
|
ABCDataFrame,
|
ABCExtensionArray,
|
ABCIndex,
|
ABCMultiIndex,
|
ABCSeries,
|
)
|
from pandas.core.dtypes.inference import is_list_like
|
|
if TYPE_CHECKING:
|
from pandas._typing import (
|
ArrayLike,
|
DtypeObj,
|
NDFrame,
|
NDFrameT,
|
Scalar,
|
npt,
|
)
|
|
from pandas.core.indexes.base import Index
|
|
|
isposinf_scalar = libmissing.isposinf_scalar
|
isneginf_scalar = libmissing.isneginf_scalar
|
|
nan_checker = np.isnan
|
INF_AS_NA = False
|
_dtype_object = np.dtype("object")
|
_dtype_str = np.dtype(str)
|
|
|
@overload
|
def isna(obj: Scalar) -> bool:
|
...
|
|
|
@overload
|
def isna(
|
obj: ArrayLike | Index | list,
|
) -> npt.NDArray[np.bool_]:
|
...
|
|
|
@overload
|
def isna(obj: NDFrameT) -> NDFrameT:
|
...
|
|
|
# handle unions
|
@overload
|
def isna(obj: NDFrameT | ArrayLike | Index | list) -> NDFrameT | npt.NDArray[np.bool_]:
|
...
|
|
|
@overload
|
def isna(obj: object) -> bool | npt.NDArray[np.bool_] | NDFrame:
|
...
|
|
|
def isna(obj: object) -> bool | npt.NDArray[np.bool_] | NDFrame:
|
"""
|
Detect missing values for an array-like object.
|
|
This function takes a scalar or array-like object and indicates
|
whether values are missing (``NaN`` in numeric arrays, ``None`` or ``NaN``
|
in object arrays, ``NaT`` in datetimelike).
|
|
Parameters
|
----------
|
obj : scalar or array-like
|
Object to check for null or missing values.
|
|
Returns
|
-------
|
bool or array-like of bool
|
For scalar input, returns a scalar boolean.
|
For array input, returns an array of boolean indicating whether each
|
corresponding element is missing.
|
|
See Also
|
--------
|
notna : Boolean inverse of pandas.isna.
|
Series.isna : Detect missing values in a Series.
|
DataFrame.isna : Detect missing values in a DataFrame.
|
Index.isna : Detect missing values in an Index.
|
|
Examples
|
--------
|
Scalar arguments (including strings) result in a scalar boolean.
|
|
>>> pd.isna('dog')
|
False
|
|
>>> pd.isna(pd.NA)
|
True
|
|
>>> pd.isna(np.nan)
|
True
|
|
ndarrays result in an ndarray of booleans.
|
|
>>> array = np.array([[1, np.nan, 3], [4, 5, np.nan]])
|
>>> array
|
array([[ 1., nan, 3.],
|
[ 4., 5., nan]])
|
>>> pd.isna(array)
|
array([[False, True, False],
|
[False, False, True]])
|
|
For indexes, an ndarray of booleans is returned.
|
|
>>> index = pd.DatetimeIndex(["2017-07-05", "2017-07-06", None,
|
... "2017-07-08"])
|
>>> index
|
DatetimeIndex(['2017-07-05', '2017-07-06', 'NaT', '2017-07-08'],
|
dtype='datetime64[ns]', freq=None)
|
>>> pd.isna(index)
|
array([False, False, True, False])
|
|
For Series and DataFrame, the same type is returned, containing booleans.
|
|
>>> df = pd.DataFrame([['ant', 'bee', 'cat'], ['dog', None, 'fly']])
|
>>> df
|
0 1 2
|
0 ant bee cat
|
1 dog None fly
|
>>> pd.isna(df)
|
0 1 2
|
0 False False False
|
1 False True False
|
|
>>> pd.isna(df[1])
|
0 False
|
1 True
|
Name: 1, dtype: bool
|
"""
|
return _isna(obj)
|
|
|
isnull = isna
|
|
|
def _isna(obj, inf_as_na: bool = False):
|
"""
|
Detect missing values, treating None, NaN or NA as null. Infinite
|
values will also be treated as null if inf_as_na is True.
|
|
Parameters
|
----------
|
obj: ndarray or object value
|
Input array or scalar value.
|
inf_as_na: bool
|
Whether to treat infinity as null.
|
|
Returns
|
-------
|
boolean ndarray or boolean
|
"""
|
if is_scalar(obj):
|
return libmissing.checknull(obj, inf_as_na=inf_as_na)
|
elif isinstance(obj, ABCMultiIndex):
|
raise NotImplementedError("isna is not defined for MultiIndex")
|
elif isinstance(obj, type):
|
return False
|
elif isinstance(obj, (np.ndarray, ABCExtensionArray)):
|
return _isna_array(obj, inf_as_na=inf_as_na)
|
elif isinstance(obj, ABCIndex):
|
# Try to use cached isna, which also short-circuits for integer dtypes
|
# and avoids materializing RangeIndex._values
|
if not obj._can_hold_na:
|
return obj.isna()
|
return _isna_array(obj._values, inf_as_na=inf_as_na)
|
|
elif isinstance(obj, ABCSeries):
|
result = _isna_array(obj._values, inf_as_na=inf_as_na)
|
# box
|
result = obj._constructor(result, index=obj.index, name=obj.name, copy=False)
|
return result
|
elif isinstance(obj, ABCDataFrame):
|
return obj.isna()
|
elif isinstance(obj, list):
|
return _isna_array(np.asarray(obj, dtype=object), inf_as_na=inf_as_na)
|
elif hasattr(obj, "__array__"):
|
return _isna_array(np.asarray(obj), inf_as_na=inf_as_na)
|
else:
|
return False
|
|
|
def _use_inf_as_na(key) -> None:
|
"""
|
Option change callback for na/inf behaviour.
|
|
Choose which replacement for numpy.isnan / -numpy.isfinite is used.
|
|
Parameters
|
----------
|
flag: bool
|
True means treat None, NaN, INF, -INF as null (old way),
|
False means None and NaN are null, but INF, -INF are not null
|
(new way).
|
|
Notes
|
-----
|
This approach to setting global module values is discussed and
|
approved here:
|
|
* https://stackoverflow.com/questions/4859217/
|
programmatically-creating-variables-in-python/4859312#4859312
|
"""
|
inf_as_na = get_option(key)
|
globals()["_isna"] = partial(_isna, inf_as_na=inf_as_na)
|
if inf_as_na:
|
globals()["nan_checker"] = lambda x: ~np.isfinite(x)
|
globals()["INF_AS_NA"] = True
|
else:
|
globals()["nan_checker"] = np.isnan
|
globals()["INF_AS_NA"] = False
|
|
|
def _isna_array(values: ArrayLike, inf_as_na: bool = False):
|
"""
|
Return an array indicating which values of the input array are NaN / NA.
|
|
Parameters
|
----------
|
obj: ndarray or ExtensionArray
|
The input array whose elements are to be checked.
|
inf_as_na: bool
|
Whether or not to treat infinite values as NA.
|
|
Returns
|
-------
|
array-like
|
Array of boolean values denoting the NA status of each element.
|
"""
|
dtype = values.dtype
|
|
if not isinstance(values, np.ndarray):
|
# i.e. ExtensionArray
|
if inf_as_na and is_categorical_dtype(dtype):
|
result = libmissing.isnaobj(values.to_numpy(), inf_as_na=inf_as_na)
|
else:
|
# error: Incompatible types in assignment (expression has type
|
# "Union[ndarray[Any, Any], ExtensionArraySupportsAnyAll]", variable has
|
# type "ndarray[Any, dtype[bool_]]")
|
result = values.isna() # type: ignore[assignment]
|
elif is_string_or_object_np_dtype(values.dtype):
|
result = _isna_string_dtype(values, inf_as_na=inf_as_na)
|
elif needs_i8_conversion(dtype):
|
# this is the NaT pattern
|
result = values.view("i8") == iNaT
|
else:
|
if inf_as_na:
|
result = ~np.isfinite(values)
|
else:
|
result = np.isnan(values)
|
|
return result
|
|
|
def _isna_string_dtype(values: np.ndarray, inf_as_na: bool) -> npt.NDArray[np.bool_]:
|
# Working around NumPy ticket 1542
|
dtype = values.dtype
|
|
if dtype.kind in ("S", "U"):
|
result = np.zeros(values.shape, dtype=bool)
|
else:
|
if values.ndim in {1, 2}:
|
result = libmissing.isnaobj(values, inf_as_na=inf_as_na)
|
else:
|
# 0-D, reached via e.g. mask_missing
|
result = libmissing.isnaobj(values.ravel(), inf_as_na=inf_as_na)
|
result = result.reshape(values.shape)
|
|
return result
|
|
|
@overload
|
def notna(obj: Scalar) -> bool:
|
...
|
|
|
@overload
|
def notna(
|
obj: ArrayLike | Index | list,
|
) -> npt.NDArray[np.bool_]:
|
...
|
|
|
@overload
|
def notna(obj: NDFrameT) -> NDFrameT:
|
...
|
|
|
# handle unions
|
@overload
|
def notna(obj: NDFrameT | ArrayLike | Index | list) -> NDFrameT | npt.NDArray[np.bool_]:
|
...
|
|
|
@overload
|
def notna(obj: object) -> bool | npt.NDArray[np.bool_] | NDFrame:
|
...
|
|
|
def notna(obj: object) -> bool | npt.NDArray[np.bool_] | NDFrame:
|
"""
|
Detect non-missing values for an array-like object.
|
|
This function takes a scalar or array-like object and indicates
|
whether values are valid (not missing, which is ``NaN`` in numeric
|
arrays, ``None`` or ``NaN`` in object arrays, ``NaT`` in datetimelike).
|
|
Parameters
|
----------
|
obj : array-like or object value
|
Object to check for *not* null or *non*-missing values.
|
|
Returns
|
-------
|
bool or array-like of bool
|
For scalar input, returns a scalar boolean.
|
For array input, returns an array of boolean indicating whether each
|
corresponding element is valid.
|
|
See Also
|
--------
|
isna : Boolean inverse of pandas.notna.
|
Series.notna : Detect valid values in a Series.
|
DataFrame.notna : Detect valid values in a DataFrame.
|
Index.notna : Detect valid values in an Index.
|
|
Examples
|
--------
|
Scalar arguments (including strings) result in a scalar boolean.
|
|
>>> pd.notna('dog')
|
True
|
|
>>> pd.notna(pd.NA)
|
False
|
|
>>> pd.notna(np.nan)
|
False
|
|
ndarrays result in an ndarray of booleans.
|
|
>>> array = np.array([[1, np.nan, 3], [4, 5, np.nan]])
|
>>> array
|
array([[ 1., nan, 3.],
|
[ 4., 5., nan]])
|
>>> pd.notna(array)
|
array([[ True, False, True],
|
[ True, True, False]])
|
|
For indexes, an ndarray of booleans is returned.
|
|
>>> index = pd.DatetimeIndex(["2017-07-05", "2017-07-06", None,
|
... "2017-07-08"])
|
>>> index
|
DatetimeIndex(['2017-07-05', '2017-07-06', 'NaT', '2017-07-08'],
|
dtype='datetime64[ns]', freq=None)
|
>>> pd.notna(index)
|
array([ True, True, False, True])
|
|
For Series and DataFrame, the same type is returned, containing booleans.
|
|
>>> df = pd.DataFrame([['ant', 'bee', 'cat'], ['dog', None, 'fly']])
|
>>> df
|
0 1 2
|
0 ant bee cat
|
1 dog None fly
|
>>> pd.notna(df)
|
0 1 2
|
0 True True True
|
1 True False True
|
|
>>> pd.notna(df[1])
|
0 True
|
1 False
|
Name: 1, dtype: bool
|
"""
|
res = isna(obj)
|
if isinstance(res, bool):
|
return not res
|
return ~res
|
|
|
notnull = notna
|
|
|
def isna_compat(arr, fill_value=np.nan) -> bool:
|
"""
|
Parameters
|
----------
|
arr: a numpy array
|
fill_value: fill value, default to np.nan
|
|
Returns
|
-------
|
True if we can fill using this fill_value
|
"""
|
if isna(fill_value):
|
dtype = arr.dtype
|
return not (is_bool_dtype(dtype) or is_integer_dtype(dtype))
|
return True
|
|
|
def array_equivalent(
|
left,
|
right,
|
strict_nan: bool = False,
|
dtype_equal: bool = False,
|
) -> bool:
|
"""
|
True if two arrays, left and right, have equal non-NaN elements, and NaNs
|
in corresponding locations. False otherwise. It is assumed that left and
|
right are NumPy arrays of the same dtype. The behavior of this function
|
(particularly with respect to NaNs) is not defined if the dtypes are
|
different.
|
|
Parameters
|
----------
|
left, right : ndarrays
|
strict_nan : bool, default False
|
If True, consider NaN and None to be different.
|
dtype_equal : bool, default False
|
Whether `left` and `right` are known to have the same dtype
|
according to `is_dtype_equal`. Some methods like `BlockManager.equals`.
|
require that the dtypes match. Setting this to ``True`` can improve
|
performance, but will give different results for arrays that are
|
equal but different dtypes.
|
|
Returns
|
-------
|
b : bool
|
Returns True if the arrays are equivalent.
|
|
Examples
|
--------
|
>>> array_equivalent(
|
... np.array([1, 2, np.nan]),
|
... np.array([1, 2, np.nan]))
|
True
|
>>> array_equivalent(
|
... np.array([1, np.nan, 2]),
|
... np.array([1, 2, np.nan]))
|
False
|
"""
|
left, right = np.asarray(left), np.asarray(right)
|
|
# shape compat
|
if left.shape != right.shape:
|
return False
|
|
if dtype_equal:
|
# fastpath when we require that the dtypes match (Block.equals)
|
if left.dtype.kind in ["f", "c"]:
|
return _array_equivalent_float(left, right)
|
elif needs_i8_conversion(left.dtype):
|
return _array_equivalent_datetimelike(left, right)
|
elif is_string_or_object_np_dtype(left.dtype):
|
# TODO: fastpath for pandas' StringDtype
|
return _array_equivalent_object(left, right, strict_nan)
|
else:
|
return np.array_equal(left, right)
|
|
# Slow path when we allow comparing different dtypes.
|
# Object arrays can contain None, NaN and NaT.
|
# string dtypes must be come to this path for NumPy 1.7.1 compat
|
if left.dtype.kind in "OSU" or right.dtype.kind in "OSU":
|
# Note: `in "OSU"` is non-trivially faster than `in ["O", "S", "U"]`
|
# or `in ("O", "S", "U")`
|
return _array_equivalent_object(left, right, strict_nan)
|
|
# NaNs can occur in float and complex arrays.
|
if is_float_dtype(left.dtype) or is_complex_dtype(left.dtype):
|
if not (left.size and right.size):
|
return True
|
return ((left == right) | (isna(left) & isna(right))).all()
|
|
elif needs_i8_conversion(left.dtype) or needs_i8_conversion(right.dtype):
|
# datetime64, timedelta64, Period
|
if not is_dtype_equal(left.dtype, right.dtype):
|
return False
|
|
left = left.view("i8")
|
right = right.view("i8")
|
|
# if we have structured dtypes, compare first
|
if (
|
left.dtype.type is np.void or right.dtype.type is np.void
|
) and left.dtype != right.dtype:
|
return False
|
|
return np.array_equal(left, right)
|
|
|
def _array_equivalent_float(left, right) -> bool:
|
return bool(((left == right) | (np.isnan(left) & np.isnan(right))).all())
|
|
|
def _array_equivalent_datetimelike(left, right):
|
return np.array_equal(left.view("i8"), right.view("i8"))
|
|
|
def _array_equivalent_object(left: np.ndarray, right: np.ndarray, strict_nan: bool):
|
if not strict_nan:
|
# isna considers NaN and None to be equivalent.
|
|
return lib.array_equivalent_object(ensure_object(left), ensure_object(right))
|
|
for left_value, right_value in zip(left, right):
|
if left_value is NaT and right_value is not NaT:
|
return False
|
|
elif left_value is libmissing.NA and right_value is not libmissing.NA:
|
return False
|
|
elif isinstance(left_value, float) and np.isnan(left_value):
|
if not isinstance(right_value, float) or not np.isnan(right_value):
|
return False
|
else:
|
try:
|
if np.any(np.asarray(left_value != right_value)):
|
return False
|
except TypeError as err:
|
if "boolean value of NA is ambiguous" in str(err):
|
return False
|
raise
|
except ValueError:
|
# numpy can raise a ValueError if left and right cannot be
|
# compared (e.g. nested arrays)
|
return False
|
return True
|
|
|
def array_equals(left: ArrayLike, right: ArrayLike) -> bool:
|
"""
|
ExtensionArray-compatible implementation of array_equivalent.
|
"""
|
if not is_dtype_equal(left.dtype, right.dtype):
|
return False
|
elif isinstance(left, ABCExtensionArray):
|
return left.equals(right)
|
else:
|
return array_equivalent(left, right, dtype_equal=True)
|
|
|
def infer_fill_value(val):
|
"""
|
infer the fill value for the nan/NaT from the provided
|
scalar/ndarray/list-like if we are a NaT, return the correct dtyped
|
element to provide proper block construction
|
"""
|
if not is_list_like(val):
|
val = [val]
|
val = np.array(val, copy=False)
|
if needs_i8_conversion(val.dtype):
|
return np.array("NaT", dtype=val.dtype)
|
elif is_object_dtype(val.dtype):
|
dtype = lib.infer_dtype(ensure_object(val), skipna=False)
|
if dtype in ["datetime", "datetime64"]:
|
return np.array("NaT", dtype=DT64NS_DTYPE)
|
elif dtype in ["timedelta", "timedelta64"]:
|
return np.array("NaT", dtype=TD64NS_DTYPE)
|
return np.nan
|
|
|
def maybe_fill(arr: np.ndarray) -> np.ndarray:
|
"""
|
Fill numpy.ndarray with NaN, unless we have a integer or boolean dtype.
|
"""
|
if arr.dtype.kind not in ("u", "i", "b"):
|
arr.fill(np.nan)
|
return arr
|
|
|
def na_value_for_dtype(dtype: DtypeObj, compat: bool = True):
|
"""
|
Return a dtype compat na value
|
|
Parameters
|
----------
|
dtype : string / dtype
|
compat : bool, default True
|
|
Returns
|
-------
|
np.dtype or a pandas dtype
|
|
Examples
|
--------
|
>>> na_value_for_dtype(np.dtype('int64'))
|
0
|
>>> na_value_for_dtype(np.dtype('int64'), compat=False)
|
nan
|
>>> na_value_for_dtype(np.dtype('float64'))
|
nan
|
>>> na_value_for_dtype(np.dtype('bool'))
|
False
|
>>> na_value_for_dtype(np.dtype('datetime64[ns]'))
|
numpy.datetime64('NaT')
|
"""
|
|
if isinstance(dtype, ExtensionDtype):
|
return dtype.na_value
|
elif needs_i8_conversion(dtype):
|
return dtype.type("NaT", "ns")
|
elif is_float_dtype(dtype):
|
return np.nan
|
elif is_integer_dtype(dtype):
|
if compat:
|
return 0
|
return np.nan
|
elif is_bool_dtype(dtype):
|
if compat:
|
return False
|
return np.nan
|
return np.nan
|
|
|
def remove_na_arraylike(arr):
|
"""
|
Return array-like containing only true/non-NaN values, possibly empty.
|
"""
|
if is_extension_array_dtype(arr):
|
return arr[notna(arr)]
|
else:
|
return arr[notna(np.asarray(arr))]
|
|
|
def is_valid_na_for_dtype(obj, dtype: DtypeObj) -> bool:
|
"""
|
isna check that excludes incompatible dtypes
|
|
Parameters
|
----------
|
obj : object
|
dtype : np.datetime64, np.timedelta64, DatetimeTZDtype, or PeriodDtype
|
|
Returns
|
-------
|
bool
|
"""
|
if not lib.is_scalar(obj) or not isna(obj):
|
return False
|
elif dtype.kind == "M":
|
if isinstance(dtype, np.dtype):
|
# i.e. not tzaware
|
return not isinstance(obj, (np.timedelta64, Decimal))
|
# we have to rule out tznaive dt64("NaT")
|
return not isinstance(obj, (np.timedelta64, np.datetime64, Decimal))
|
elif dtype.kind == "m":
|
return not isinstance(obj, (np.datetime64, Decimal))
|
elif dtype.kind in ["i", "u", "f", "c"]:
|
# Numeric
|
return obj is not NaT and not isinstance(obj, (np.datetime64, np.timedelta64))
|
elif dtype.kind == "b":
|
# We allow pd.NA, None, np.nan in BooleanArray (same as IntervalDtype)
|
return lib.is_float(obj) or obj is None or obj is libmissing.NA
|
|
elif dtype == _dtype_str:
|
# numpy string dtypes to avoid float np.nan
|
return not isinstance(obj, (np.datetime64, np.timedelta64, Decimal, float))
|
|
elif dtype == _dtype_object:
|
# This is needed for Categorical, but is kind of weird
|
return True
|
|
elif isinstance(dtype, PeriodDtype):
|
return not isinstance(obj, (np.datetime64, np.timedelta64, Decimal))
|
|
elif isinstance(dtype, IntervalDtype):
|
return lib.is_float(obj) or obj is None or obj is libmissing.NA
|
|
elif isinstance(dtype, CategoricalDtype):
|
return is_valid_na_for_dtype(obj, dtype.categories.dtype)
|
|
# fallback, default to allowing NaN, None, NA, NaT
|
return not isinstance(obj, (np.datetime64, np.timedelta64, Decimal))
|
|
|
def isna_all(arr: ArrayLike) -> bool:
|
"""
|
Optimized equivalent to isna(arr).all()
|
"""
|
total_len = len(arr)
|
|
# Usually it's enough to check but a small fraction of values to see if
|
# a block is NOT null, chunks should help in such cases.
|
# parameters 1000 and 40 were chosen arbitrarily
|
chunk_len = max(total_len // 40, 1000)
|
|
dtype = arr.dtype
|
if dtype.kind == "f" and isinstance(dtype, np.dtype):
|
checker = nan_checker
|
|
elif (isinstance(dtype, np.dtype) and dtype.kind in ["m", "M"]) or isinstance(
|
dtype, (DatetimeTZDtype, PeriodDtype)
|
):
|
# error: Incompatible types in assignment (expression has type
|
# "Callable[[Any], Any]", variable has type "ufunc")
|
checker = lambda x: np.asarray(x.view("i8")) == iNaT # type: ignore[assignment]
|
|
else:
|
# error: Incompatible types in assignment (expression has type "Callable[[Any],
|
# Any]", variable has type "ufunc")
|
checker = lambda x: _isna_array( # type: ignore[assignment]
|
x, inf_as_na=INF_AS_NA
|
)
|
|
return all(
|
checker(arr[i : i + chunk_len]).all() for i in range(0, total_len, chunk_len)
|
)
|
