adding support for Min/Max over LargeList and FixedSizeList

datafusion/common/src/scalar/mod.rs

@@ -4941,6 +4941,40 @@ mod tests {
                 ])]),
             ));
         assert_eq!(a.partial_cmp(&b), Some(Ordering::Greater));
+
+        let a = ScalarValue::LargeList(Arc::new(LargeListArray::from_iter_primitive::<
+            Int64Type,
+            _,
+            _,
+        >(vec![Some(vec![
+            None,
+            Some(2),
+            Some(3),
+        ])])));
+        let b = ScalarValue::LargeList(Arc::new(LargeListArray::from_iter_primitive::<
+            Int64Type,
+            _,
+            _,
+        >(vec![Some(vec![
+            Some(1),
+            Some(2),
+            Some(3),
+        ])])));
+        assert_eq!(a.partial_cmp(&b), Some(Ordering::Greater));
+
+        let a = ScalarValue::FixedSizeList(Arc::new(
+            FixedSizeListArray::from_iter_primitive::<Int64Type, _, _>(
+                vec![Some(vec![None, Some(2), Some(3)])],
+                3,
+            ),
+        ));
+        let b = ScalarValue::FixedSizeList(Arc::new(
+            FixedSizeListArray::from_iter_primitive::<Int64Type, _, _>(
+                vec![Some(vec![Some(1), Some(2), Some(3)])],
+                3,
+            ),
+        ));
+        assert_eq!(a.partial_cmp(&b), Some(Ordering::Greater));
     }
 
     #[test]

datafusion/functions-aggregate/src/min_max.rs

@@ -622,6 +622,10 @@ fn min_batch(values: &ArrayRef) -> Result<ScalarValue> {
         }
         DataType::Struct(_) => min_max_batch_generic(values, Ordering::Greater)?,
         DataType::List(_) => min_max_batch_generic(values, Ordering::Greater)?,
+        DataType::LargeList(_) => min_max_batch_generic(values, Ordering::Greater)?,
+        DataType::FixedSizeList(_, _) => {
+            min_max_batch_generic(values, Ordering::Greater)?
+        }
         DataType::Dictionary(_, _) => {
             let values = values.as_any_dictionary().values();
             min_batch(values)?
@@ -720,6 +724,8 @@ pub fn max_batch(values: &ArrayRef) -> Result<ScalarValue> {
         }
         DataType::Struct(_) => min_max_batch_generic(values, Ordering::Less)?,
         DataType::List(_) => min_max_batch_generic(values, Ordering::Less)?,
+        DataType::LargeList(_) => min_max_batch_generic(values, Ordering::Less)?,
+        DataType::FixedSizeList(_, _) => min_max_batch_generic(values, Ordering::Less)?,
         DataType::Dictionary(_, _) => {
             let values = values.as_any_dictionary().values();
             max_batch(values)?
@@ -1008,6 +1014,23 @@ macro_rules! min_max {
             ) => {
                 min_max_generic!(lhs, rhs, $OP)
             }
+
+
+            (
+                lhs @ ScalarValue::LargeList(_),
+                rhs @ ScalarValue::LargeList(_),
+            ) => {
+                min_max_generic!(lhs, rhs, $OP)
+            }
+
+
+            (
+                lhs @ ScalarValue::FixedSizeList(_),
+                rhs @ ScalarValue::FixedSizeList(_),
+            ) => {
+                min_max_generic!(lhs, rhs, $OP)
+            }
+
             e => {
                 return internal_err!(
                     "MIN/MAX is not expected to receive scalars of incompatible types {:?}",

datafusion/optimizer/src/analyzer/type_coercion.rs

@@ -719,6 +719,8 @@ fn extract_window_frame_target_type(col_type: &DataType) -> Result<DataType> {
     if col_type.is_numeric()
         || is_utf8_or_utf8view_or_large_utf8(col_type)
         || matches!(col_type, DataType::List(_))
+        || matches!(col_type, DataType::LargeList(_))
+        || matches!(col_type, DataType::FixedSizeList(_, _))
         || matches!(col_type, DataType::Null)
         || matches!(col_type, DataType::Boolean)
     {

datafusion/sqllogictest/test_files/aggregate.slt

@@ -7018,151 +7018,3 @@ VALUES
 );
 ----
 {a: 1, b: 2, c: 3} {a: 1, b: 2, c: 4}
-
-# Min/Max with list over integers
-query ??
-SELECT MIN(column1), MAX(column1) FROM VALUES
-([1, 2, 3]),
-([1, 2]);
-----
-[1, 2] [1, 2, 3]
-
-# Min/Max with lists over strings
-query ??
-SELECT MIN(column1), MAX(column1) FROM VALUES
-(['a', 'b', 'c']),
-(['a', 'b']);
-----
-[a, b] [a, b, c]
-
-# Min/Max with list over booleans
-query ??
-SELECT MIN(column1), MAX(column1) FROM VALUES
-([true, true, false]),
-([false, true]);
-----
-[false, true] [true, true, false]
-
-# Min/Max with list over nullable integers
-query ??
-SELECT MIN(column1), MAX(column1) FROM VALUES
-([NULL, 1, 2]),
-([1, 2]);
-----
-[1, 2] [NULL, 1, 2]
-
-# Min/Max list with different lengths and nulls
-query ??
-SELECT MIN(column1), MAX(column1) FROM VALUES
-([1, NULL, 3]),
-([1, 2, 3, 4]),
-([1, 2]);
-----
-[1, 2] [1, NULL, 3]
-
-# Min/Max list with only NULLs
-query ??
-SELECT MIN(column1), MAX(column1) FROM VALUES
-([NULL, NULL]),
-([NULL]);
-----
-[NULL] [NULL, NULL]
-
-# Min/Max list with empty lists
-query ??
-SELECT MIN(column1), MAX(column1) FROM VALUES
-([]),
-([1]),
-([]);
-----
-[] [1]
-
-# Min/Max list of varying types (integers and NULLs)
-query ??
-SELECT MIN(column1), MAX(column1) FROM VALUES
-([1, 2, 3]),
-([NULL, 2, 3]),
-([1, 2, NULL]);
-----
-[1, 2, 3] [NULL, 2, 3]
-
-# Min/Max list grouped by key with NULLs and differing lengths
-query I?? rowsort
-SELECT column1, MIN(column2), MAX(column2) FROM VALUES
-(0, [1, NULL, 3]),
-(0, [1, 2, 3, 4]),
-(1, [1, 2]),
-(1, [NULL, 5]),
-(1, [])
-GROUP BY column1;
-----
-0 [1, 2, 3, 4] [1, NULL, 3]
-1 [] [NULL, 5]
-
-# Min/Max list grouped by key with NULLs and differing lengths
-query I?? rowsort
-SELECT column1, MIN(column2), MAX(column2) FROM VALUES
-(0, [NULL]),
-(0, [NULL, NULL]),
-(1, [NULL])
-GROUP BY column1;
-----
-0 [NULL] [NULL, NULL]
-1 [NULL] [NULL]
-
-# Min/Max grouped list with empty and non-empty
-query I?? rowsort
-SELECT column1, MIN(column2), MAX(column2) FROM VALUES
-(0, []),
-(0, [1]),
-(0, []),
-(1, [5, 6]),
-(1, [])
-GROUP BY column1;
-----
-0 [] [1]
-1 [] [5, 6]
-
-# Min/Max over lists with a window function
-query ?
-SELECT min(column1) OVER (ORDER BY column1) FROM VALUES
-([1, 2, 3]),
-([1, 2, 3]),
-([2, 3])
-----
-[1, 2, 3]
-[1, 2, 3]
-[1, 2, 3]
-
-# Min/Max over lists with a window function and nulls
-query ?
-SELECT min(column1) OVER (ORDER BY column1) FROM VALUES
-(NULL),
-([4, 5]),
-([2, 3])
-----
-[2, 3]
-[2, 3]
-[2, 3]
-
-# Min/Max over lists with a window function, nulls and ROWS BETWEEN statement
-query ?
-SELECT min(column1) OVER (ORDER BY column1 ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) FROM VALUES
-(NULL),
-([4, 5]),
-([2, 3])
-----
-[2, 3]
-[2, 3]
-[4, 5]
-
-# Min/Max over lists with a window function using a different column
-query ?
-SELECT max(column2) OVER (ORDER BY column1 ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) FROM VALUES
-([1, 2, 3], [4, 5]),
-([2, 3], [2, 3]),
-([1, 2, 3], NULL)
-----
-[4, 5]
-[4, 5]
-[2, 3]

datafusion/sqllogictest/test_files/min_max/fixed_size_list.slt

@@ -0,0 +1,134 @@
+# Min/Max with FixedSizeList over integers
+query ??
+SELECT MIN(column1), MAX(column1) FROM VALUES
+(arrow_cast(make_array(1, 2, 3, 4), 'FixedSizeList(4, Int64)')),
+(arrow_cast(make_array(1, 2), 'FixedSizeList(2, Int64)'));
+----
+[1, 2] [1, 2, 3, 4]
+
+# Min/Max with FixedSizeList over strings
+query ??
+SELECT MIN(column1), MAX(column1) FROM VALUES
+(arrow_cast(make_array('a', 'b', 'c'), 'FixedSizeList(3, Utf8)')),
+(arrow_cast(make_array('a', 'b'), 'LargeList(Utf8)'));
+----
+[a, b] [a, b, c]
+
+# Min/Max with FixedSizeList over booleans
+query ??
+SELECT MIN(column1), MAX(column1) FROM VALUES
+(arrow_cast(make_array(true, false, true), 'FixedSizeList(3, Boolean)')),
+(arrow_cast(make_array(true, false), 'FixedSizeList(2, Boolean)'));
+----
+[true, false] [true, false, true]
+
+# Min/Max with FixedSizeList over nullable integers
+query ??
+SELECT MIN(column1), MAX(column1) FROM VALUES
+(arrow_cast(make_array(NULL, 1, 2), 'FixedSizeList(3, Int64)')),
+(arrow_cast(make_array(1, 2), 'FixedSizeList(2, Int64)'));
+----
+[1, 2] [NULL, 1, 2]
+
+# Min/Max FixedSizeList with different lengths and nulls
+query ??
+SELECT MIN(column1), MAX(column1) FROM VALUES
+(arrow_cast(make_array(1, 2, 3, 4), 'FixedSizeList(4, Int64)')),
+(arrow_cast(make_array(1, 2), 'FixedSizeList(2, Int64)')),
+(arrow_cast(make_array(1, NULL, 3), 'FixedSizeList(3, Int64)'));
+----
+[1, 2] [1, NULL, 3]
+
+# Min/Max FixedSizeList with only NULLs
+query ??
+SELECT MIN(column1), MAX(column1) FROM VALUES
+(arrow_cast(make_array(NULL, NULL), 'FixedSizeList(2, Int64)')),
+(arrow_cast(make_array(NULL), 'FixedSizeList(1, Int64)'));
+----
+[NULL] [NULL, NULL]
+
+
+# Min/Max FixedSizeList of varying types (integers and NULLs)
+query ??
+SELECT MIN(column1), MAX(column1) FROM VALUES
+(arrow_cast(make_array(1, 2, 3), 'FixedSizeList(3, Int64)')),
+(arrow_cast(make_array(NULL, 2, 3), 'FixedSizeList(3, Int64)')),
+(arrow_cast(make_array(1, 2, NULL), 'FixedSizeList(3, Int64)'));
+----
+[1, 2, 3] [NULL, 2, 3]
+
+# Min/Max FixedSizeList grouped by key with NULLs and differing lengths
+query I?? rowsort
+SELECT column1, MIN(column2), MAX(column2) FROM VALUES
+(0, arrow_cast(make_array(1, NULL, 3), 'FixedSizeList(3, Int64)')),
+(0, arrow_cast(make_array(1, 2, 3, 4), 'FixedSizeList(4, Int64)')),
+(1, arrow_cast(make_array(1, 2), 'FixedSizeList(2, Int64)')),
+(1, arrow_cast(make_array(NULL, 5), 'FixedSizeList(2, Int64)'))
+GROUP BY column1;
+----
+0 [1, 2, 3, 4] [1, NULL, 3]
+1 [1, 2] [NULL, 5]
+
+# Min/Max FixedSizeList grouped by key with NULLs and differing lengths
+query I?? rowsort
+SELECT column1, MIN(column2), MAX(column2) FROM VALUES
+(0, arrow_cast(make_array(NULL), 'FixedSizeList(1, Int64)')),
+(0, arrow_cast(make_array(NULL, NULL), 'FixedSizeList(2, Int64)')),
+(1, arrow_cast(make_array(NULL), 'FixedSizeList(1, Int64)'))
+GROUP BY column1;
+----
+0 [NULL] [NULL, NULL]
+1 [NULL] [NULL]
+
+# Min/Max grouped FixedSizeList with empty and non-empty
+query I?? rowsort
+SELECT column1, MIN(column2), MAX(column2) FROM VALUES
+(0, arrow_cast(make_array(1), 'FixedSizeList(1, Int64)')),
+(1, arrow_cast(make_array(5, 6), 'FixedSizeList(2, Int64)'))
+GROUP BY column1;
+----
+0 [1] [1]
+1 [5, 6] [5, 6]
+
+# Min/Max over FixedSizeList with a window function
+query ?
+SELECT min(column1) OVER (ORDER BY column1) FROM VALUES
+(arrow_cast(make_array(1, 2, 3), 'FixedSizeList(3, Int64)')),
+(arrow_cast(make_array(1, 2, 3), 'FixedSizeList(3, Int64)')),
+(arrow_cast(make_array(2, 3), 'FixedSizeList(2, Int64)'))
+----
+[1, 2, 3]
+[1, 2, 3]
+[1, 2, 3]
+
+# Min/Max over FixedSizeList with a window function and nulls
+query ?
+SELECT min(column1) OVER (ORDER BY column1) FROM VALUES
+(arrow_cast(make_array(NULL), 'FixedSizeList(1, Int64)')),
+(arrow_cast(make_array(4, 5), 'FixedSizeList(2, Int64)')),
+(arrow_cast(make_array(2, 3), 'FixedSizeList(2, Int64)'))
+----
+[2, 3]
+[2, 3]
+[2, 3]
+
+# Min/Max over FixedSizeList with a window function, nulls and ROWS BETWEEN statement
+query ?
+SELECT min(column1) OVER (ORDER BY column1 ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) FROM VALUES
+(arrow_cast(make_array(NULL), 'FixedSizeList(1, Int64)')),
+(arrow_cast(make_array(4, 5), 'FixedSizeList(2, Int64)')),
+(arrow_cast(make_array(2, 3), 'FixedSizeList(2, Int64)'))
+----
+[2, 3]
+[2, 3]
+[4, 5]
+
+# Min/Max over FixedSizeList with a window function using a different column
+query ?
+SELECT max(column2) OVER (ORDER BY column1 ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) FROM VALUES
+(arrow_cast(make_array(1, 2, 3), 'FixedSizeList(3, Int64)'), arrow_cast(make_array(4, 5), 'FixedSizeList(2, Int64)')),
+(arrow_cast(make_array(2, 3), 'FixedSizeList(2, Int64)'), arrow_cast(make_array(2, 3), 'FixedSizeList(2, Int64)'))
+----
+[4, 5]
+[4, 5]
+