Merge pull request #2 from edponce/ARROW-12959-R-Option-for-is-nullNaN-to-evaluate-to-t

Christian8491 · web-flow · commit 28310635d7c2 · 2021-08-18T11:08:03.000-05:00
ARROW-12959: [C++] Option for is_null(NaN) to evaluate to true
diff --git a/cpp/src/arrow/compute/kernels/scalar_validity.cc b/cpp/src/arrow/compute/kernels/scalar_validity.cc
@@ -32,9 +32,6 @@ namespace compute {
 namespace internal {
 namespace {
 
-template <typename T, typename R = T>
-using enable_if_floating_point = enable_if_t<std::is_floating_point<T>::value, R>;
-
 struct IsValidOperator {
   static Status Call(KernelContext* ctx, const Scalar& in, Scalar* out) {
     checked_cast<BooleanScalar*>(out)->value = in.is_valid;
@@ -78,9 +75,11 @@ struct IsInfOperator {
   }
 };
 
+using NanNullState = OptionsWrapper<NanNullOptions>;
+
 struct IsNullOperator {
   static Status Call(KernelContext* ctx, const Scalar& in, Scalar* out) {
-    const auto options = OptionsWrapper<NanNullOptions>::Get(ctx);
+    const auto options = NanNullState::Get(ctx);
     bool* out_value = &checked_cast<BooleanScalar*>(out)->value;
     if (in.is_valid) {
       if (is_floating(in.type->id())) {
@@ -98,7 +97,6 @@ struct IsNullOperator {
         }
       } else {
         *out_value = false;
-
       }
     } else {
       *out_value = true;
@@ -108,8 +106,7 @@ struct IsNullOperator {
   }
 
   template <typename T>
-  static enable_if_floating_point<T, void> SetNanBits(const ArrayData& arr,
-                                                      ArrayData* out) {
+  static void SetNanBits(const ArrayData& arr, ArrayData* out) {
     const T* data = arr.GetValues<T>(1);
     for (int64_t i = 0; i < arr.length; ++i) {
       if (std::isnan(data[i])) {
@@ -130,7 +127,7 @@ struct IsNullOperator {
     }
 
     if (is_floating(arr.type->id())) {
-      const auto options = OptionsWrapper<NanNullOptions>::Get(ctx);
+      const auto options = NanNullState::Get(ctx);
       if (options.nan_is_null) {
         switch (arr.type->id()) {
           case Type::FLOAT:
@@ -260,8 +257,8 @@ const FunctionDoc is_inf_doc(
     {"values"});
 
 const FunctionDoc is_null_doc(
-    "Return true if null, NaN can be considered as null",
-    ("For each input value, emit true iff the value is null. Default behavior is to emit "
+    "Return true if null, NaN values can be considered as null",
+    ("For each input value, emit true if the value is null. Default behavior is to emit "
      "false for NaN values. True can be emitted for NaN values by toggling "
      "NanNullOptions flag."),
     {"values"}, "NanNullOptions");
@@ -279,8 +276,7 @@ void RegisterScalarValidity(FunctionRegistry* registry) {
 
   MakeFunction("is_null", &is_null_doc, {ValueDescr::ANY}, boolean(), IsNullExec,
                registry, MemAllocation::PREALLOCATE,
-               /*can_write_into_slices=*/true, &kNanNullOptions,
-               OptionsWrapper<NanNullOptions>::Init);
+               /*can_write_into_slices=*/true, &kNanNullOptions, NanNullState::Init);
 
   DCHECK_OK(registry->AddFunction(MakeIsFiniteFunction("is_finite", &is_finite_doc)));
   DCHECK_OK(registry->AddFunction(MakeIsInfFunction("is_inf", &is_inf_doc)));
diff --git a/docs/source/cpp/compute.rst b/docs/source/cpp/compute.rst
@@ -899,33 +899,33 @@ Structural transforms
 
 .. XXX (this category is a bit of a hodgepodge)
 
-+--------------------------+------------+---------------------------------------------------+---------------------+---------+
-| Function name            | Arity      | Input types                                       | Output type         | Notes   |
-+==========================+============+===================================================+=====================+=========+
-| case_when                | Varargs    | Struct of Boolean (Arg 0), Any fixed-width (rest) | Input type          | \(1)    |
-+--------------------------+------------+---------------------------------------------------+---------------------+---------+
-| choose                   | Varargs    | Integral (Arg 0); Fixed-width/Binary-like (rest)  | Input type          | \(2)    |
-+--------------------------+------------+---------------------------------------------------+---------------------+---------+
-| coalesce                 | Varargs    | Any                                               | Input type          | \(3)    |
-+--------------------------+------------+---------------------------------------------------+---------------------+---------+
-| fill_null                | Binary     | Boolean, Null, Numeric, Temporal, String-like     | Input type          | \(4)    |
-+--------------------------+------------+---------------------------------------------------+---------------------+---------+
-| if_else                  | Ternary    | Boolean, Null, Numeric, Temporal                  | Input type          | \(5)    |
-+--------------------------+------------+---------------------------------------------------+---------------------+---------+
-| is_finite                | Unary      | Float, Double                                     | Boolean             | \(6)    |
-+--------------------------+------------+---------------------------------------------------+---------------------+---------+
-| is_inf                   | Unary      | Float, Double                                     | Boolean             | \(7)    |
-+--------------------------+------------+---------------------------------------------------+---------------------+---------+
-| is_nan                   | Unary      | Float, Double                                     | Boolean             | \(8)    |
-+--------------------------+------------+---------------------------------------------------+---------------------+---------+
-| is_null                  | Unary      | Any                                               | Boolean             | \(9)    |
-+--------------------------+------------+---------------------------------------------------+---------------------+---------+
-| is_valid                 | Unary      | Any                                               | Boolean             | \(10)   |
-+--------------------------+------------+---------------------------------------------------+---------------------+---------+
-| list_value_length        | Unary      | List-like                                         | Int32 or Int64      | \(11)   |
-+--------------------------+------------+---------------------------------------------------+---------------------+---------+
-| make_struct              | Varargs    | Any                                               | Struct              | \(12)   |
-+--------------------------+------------+---------------------------------------------------+---------------------+---------+
++-------------------+---------+---------------------------------------------------+----------------+-----------------------------+-------+
+| Function name     | Arity   | Input types                                       | Output type    | Options class               | Notes |
++===================+=========+===================================================+================+=============================+=======+
+| case_when         | Varargs | Struct of Boolean (Arg 0), Any fixed-width (rest) | Input type     |                             | \(1)  |
++-------------------+---------+---------------------------------------------------+----------------+-----------------------------+-------+
+| choose            | Varargs | Integral (Arg 0); Fixed-width/Binary-like (rest)  | Input type     |                             | \(2)  |
++-------------------+---------+---------------------------------------------------+----------------+-----------------------------+-------+
+| coalesce          | Varargs | Any                                               | Input type     |                             | \(3)  |
++-------------------+---------+---------------------------------------------------+----------------+-----------------------------+-------+
+| fill_null         | Binary  | Boolean, Null, Numeric, Temporal, String-like     | Input type     |                             | \(4)  |
++-------------------+---------+---------------------------------------------------+----------------+-----------------------------+-------+
+| if_else           | Ternary | Boolean, Null, Numeric, Temporal                  | Input type     |                             | \(5)  |
++-------------------+---------+---------------------------------------------------+----------------+-----------------------------+-------+
+| is_finite         | Unary   | Float, Double                                     | Boolean        |                             | \(6)  |
++-------------------+---------+---------------------------------------------------+----------------+-----------------------------+-------+
+| is_inf            | Unary   | Float, Double                                     | Boolean        |                             | \(7)  |
++-------------------+---------+---------------------------------------------------+----------------+-----------------------------+-------+
+| is_nan            | Unary   | Float, Double                                     | Boolean        |                             | \(8)  |
++-------------------+---------+---------------------------------------------------+----------------+-----------------------------+-------+
+| is_null           | Unary   | Any                                               | Boolean        | :struct:`NanNullOptions`    | \(9)  |
++-------------------+---------+---------------------------------------------------+----------------+-----------------------------+-------+
+| is_valid          | Unary   | Any                                               | Boolean        |                             | \(10) |
++-------------------+---------+---------------------------------------------------+----------------+-----------------------------+-------+
+| list_value_length | Unary   | List-like                                         | Int32 or Int64 |                             | \(11) |
++-------------------+---------+---------------------------------------------------+----------------+-----------------------------+-------+
+| make_struct       | Varargs | Any                                               | Struct         | :struct:`MakeStructOptions` | \(12) |
++-------------------+---------+---------------------------------------------------+----------------+-----------------------------+-------+
 
 * \(1) This function acts like a SQL 'case when' statement or switch-case. The
   input is a "condition" value, which is a struct of Booleans, followed by the
@@ -966,7 +966,8 @@ Structural transforms
 
 * \(8) Output is true iff the corresponding input element is NaN.
 
-* \(9) Output is true iff the corresponding input element is null.
+* \(9) Output is true if the corresponding input element is null or if NaN
+  values are treated as null via the :struct:`NanNullOptions`.
 
 * \(10) Output is true iff the corresponding input element is non-null.
 
diff --git a/python/pyarrow/_dataset.pyx b/python/pyarrow/_dataset.pyx
@@ -238,7 +238,7 @@ cdef class Expression(_Weakrefable):
         """Checks whether the expression is null"""
         cdef:
             shared_ptr[CFunctionOptions] c_options
-        
+
         c_options.reset(new CNanNullOptions(nan_is_null))
         return Expression._call("is_null", [self], c_options)
 
diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
@@ -1039,11 +1039,12 @@ cdef class Array(_PandasConvertible):
         else:
             return 0
 
-    def is_null(self):
+    def is_null(self, nan_is_null=False):
         """
         Return BooleanArray indicating the null values.
         """
-        return _pc().is_null(self)
+        options = _pc().NanNullOptions(nan_is_null)
+        return _pc().call_function('is_null', [self], options)
 
     def is_valid(self):
         """
diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi
@@ -170,11 +170,12 @@ cdef class ChunkedArray(_PandasConvertible):
             else:
                 index -= self.chunked_array.chunk(j).get().length()
 
-    def is_null(self):
+    def is_null(self, nan_is_null=False):
         """
         Return BooleanArray indicating the null values.
         """
-        return _pc().is_null(self)
+        options = _pc().NanNullOptions(nan_is_null)
+        return _pc().call_function('is_null', [self], options)
 
     def is_valid(self):
         """
