Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve speed of median by implementing special GroupsAccumulator #13681

Open
wants to merge 18 commits into
base: main
Choose a base branch
from
+526 −2
Open

Improve speed of median by implementing special GroupsAccumulator #13681

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
d6fd21e
draft of `MedianGroupAccumulator`.
Rachelint Dec 7, 2024
91edc47
impl `state`.
Rachelint Dec 7, 2024
7838369
impl rest methods of `MedianGroupsAccumulator`.
Rachelint Dec 8, 2024
79ac655
improve comments.
Rachelint Dec 8, 2024
10b768b
use `MedianGroupsAccumulator`.
Rachelint Dec 8, 2024
311f82f
remove unused import.
Rachelint Dec 8, 2024
c86cb2e
add `group_median_table` to test group median.
Rachelint Dec 21, 2024
0957ff9
complete group median test cases in aggregate slt.
Rachelint Jan 22, 2025
a53da8c
fix type of state.
Rachelint Jan 22, 2025
34771a7
Merge branch 'main' into impl-group-accumulator-for-median
Rachelint Jan 22, 2025
e3990b8
Clippy
Dandandan Jan 23, 2025
9cd7c70
Fmt
Dandandan Jan 23, 2025
d18b347
add fuzzy tests for median.
Rachelint Jan 24, 2025
987a7d4
fix decimal.
Rachelint Jan 24, 2025
84942b3
fix clippy.
Rachelint Jan 24, 2025
40284c8
improve comments.
Rachelint Jan 24, 2025
fe5cbe0
add median cases with nulls.
Rachelint Jan 26, 2025
d8d9a19
Update datafusion/functions-aggregate/src/median.rs
Rachelint Jan 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
add group_median_table to test group median.
  • Loading branch information
@Rachelint
Rachelint committed Dec 21, 2024
commit c86cb2e534c21051505c9617afa0490c487d60ec
6 changes: 3 additions & 3 deletions datafusion/functions-aggregate/src/median.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -448,14 +448,14 @@ impl<T: ArrowNumericType + Send> GroupsAccumulator for MedianGroupsAccumulator<T
// `nulls` for converted `ListArray`
let nulls = filtered_null_mask(opt_filter, input_array);

let converted_list_array = Arc::new(ListArray::new(
let converted_list_array = ListArray::new(
Arc::new(Field::new_list_field(self.data_type.clone(), false)),
offsets,
Arc::new(values),
nulls,
));
);

Ok(vec![converted_list_array])
Ok(vec![Arc::new(converted_list_array)])
}

fn supports_convert_to_state(&self) -> bool {
Expand Down
25 changes: 25 additions & 0 deletions datafusion/sqllogictest/test_files/aggregate.slt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,31 @@ statement ok
CREATE TABLE test (c1 BIGINT,c2 BIGINT) as values
(0,null), (1,1), (null,1), (3,2), (3,2)

statement ok
CREATE TABLE group_median_table (
col_group STRING,
col_i8 TINYINT,
col_i16 SMALLINT,
col_i32 INT,
col_i64 BIGINT,
col_u8 TINYINT UNSIGNED,
col_u16 SMALLINT UNSIGNED,
col_u32 INT UNSIGNED,
col_u64 BIGINT UNSIGNED,
col_f32 FLOAT,
col_f64 DOUBLE,
col_f64_nan DOUBLE
) as VALUES
( "group0", -128, -32768, -2147483648, arrow_cast(-9223372036854775808,'Int64'), 0, 0, 0, arrow_cast(0,'UInt64'), 1.1, 1.1, 1.1 ),
( "group0", -128, -32768, -2147483648, arrow_cast(-9223372036854775808,'Int64'), 0, 0, 0, arrow_cast(0,'UInt64'), 4.4, 4.4, arrow_cast('NAN','Float64') ),
( "group0", 100, 100, 100, arrow_cast(100,'Int64'), 100, 100, 100, arrow_cast(100,'UInt64'), 3.3, 3.3, arrow_cast('NAN','Float64') ),
( "group0", 127, 32767, 2147483647, arrow_cast(9223372036854775807,'Int64'), 255, 65535, 4294967295, 18446744073709551615, 2.2, 2.2, arrow_cast('NAN','Float64') ),
( "group1", -128, -32768, -2147483648, arrow_cast(-9223372036854775808,'Int64'), 0, 0, 0, arrow_cast(0,'UInt64'), 1.1, 1.1, 1.1 ),
( "group1", -128, -32768, -2147483648, arrow_cast(-9223372036854775808,'Int64'), 0, 0, 0, arrow_cast(0,'UInt64'), 4.4, 4.4, arrow_cast('NAN','Float64') ),
( "group1", 100, 100, 100, arrow_cast(100,'Int64'), 101, 100, 100, arrow_cast(100,'UInt64'), 3.3, 3.3, arrow_cast('NAN','Float64') ),
( "group1", 125, 32766, 2147483646, arrow_cast(9223372036854775806,'Int64'), 100, 101, 4294967294, arrow_cast(100,'UInt64'), 3.2, 5.5, arrow_cast('NAN','Float64') ),
( "group1", 127, 32767, 2147483647, arrow_cast(9223372036854775807,'Int64'), 255, 65535, 4294967295, 18446744073709551615, 2.2, 2.2, arrow_cast('NAN','Float64') )

#######
# Error tests
#######
Expand Down
Loading