fix: align array_has null buffer for scalar (apache#17272)

* fix: align `array_has` null buffer for scalar

Author: comphead

datafusion/physical-expr-common/src/datum.rs

@@ -154,9 +154,26 @@ pub fn compare_op_for_nested(
     if matches!(op, Operator::IsDistinctFrom | Operator::IsNotDistinctFrom) {
         Ok(BooleanArray::new(values, None))
     } else {
-        // If one of the side is NULL, we returns NULL
+        // If one of the side is NULL, we return NULL
         // i.e. NULL eq NULL -> NULL
-        let nulls = NullBuffer::union(l.nulls(), r.nulls());
+        // For nested comparisons, we need to ensure the null buffer matches the result length
+        let nulls = match (is_l_scalar, is_r_scalar) {
+            (false, false) | (true, true) => NullBuffer::union(l.nulls(), r.nulls()),
+            (true, false) => {
+                // When left is null-scalar and right is array, expand left nulls to match result length
+                match l.nulls().filter(|nulls| !nulls.is_valid(0)) {
+                    Some(_) => Some(NullBuffer::new_null(len)), // Left scalar is null
+                    None => r.nulls().cloned(),                 // Left scalar is non-null
+                }
+            }
+            (false, true) => {
+                // When right is null-scalar and left is array, expand right nulls to match result length
+                match r.nulls().filter(|nulls| !nulls.is_valid(0)) {
+                    Some(_) => Some(NullBuffer::new_null(len)), // Right scalar is null
+                    None => l.nulls().cloned(), // Right scalar is non-null
+                }
+            }
+        };
         Ok(BooleanArray::new(values, nulls))
     }
 }

datafusion/sqllogictest/test_files/array.slt

@@ -8065,6 +8065,25 @@ FixedSizeList(Field { name: "item", data_type: Int32, nullable: true, dict_id: 0
 statement error
 create table varying_fixed_size_col_table (a int[3]) as values ([1,2,3]), ([4,5]);
 
+# https://github.com/apache/datafusion/issues/16187
+# should be NULL in case of out of bounds for Null Type
+query ?
+select [named_struct('a', 1, 'b', null)][-2];
+----
+NULL
+
+statement ok
+COPY (select [[true, false], [false, true]] a, [false, true] b union select [[null, null]], null) to 'test_files/scratch/array/array_has/single_file.parquet' stored as parquet;
+
+statement ok
+CREATE EXTERNAL TABLE array_has STORED AS PARQUET location 'test_files/scratch/array/array_has/single_file.parquet';
+
+query B
+select array_contains(a, b) from array_has order by 1 nulls last;
+----
+true
+NULL
+
 ### Delete tables
 
 statement ok
@@ -8243,3 +8262,6 @@ drop table values_all_empty;
 
 statement ok
 drop table fixed_size_col_table;
+
+statement ok
+drop table array_has;