Skip to content

Commit c67484c

Browse files
zhangfengcdtzhangfengcdt
authored
chore: Upgrade Datafusion (v50) and Arrow (v56) dependencies (#237)
1 parent 8e6234c commit c67484c

File tree

20 files changed

+389
-265
lines changed

20 files changed

+389
-265
lines changed

‎Cargo.lock‎

Lines changed: 236 additions & 207 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 23 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -56,30 +56,31 @@ rust-version = "1.82"
5656
[workspace.dependencies]
5757
approx = "0.5"
5858
abi_stable = "0.11.3"
59-
adbc_core = "0.20.0"
60-
adbc_ffi = "0.20.0"
59+
adbc_core = ">=0.20.0"
60+
adbc_ffi = ">=0.20.0"
6161
lru = "0.12"
62-
arrow = { version = "55.1.0", features = ["prettyprint", "ffi", "chrono-tz"] }
63-
arrow-array = { version = "55.1.0" }
64-
arrow-cast = { version = "55.1.0" }
65-
arrow-data = { version = "55.1.0" }
66-
arrow-json = { version = "55.1.0" }
67-
arrow-schema = { version = "55.1.0" }
62+
arrow = { version = "56.0.0", features = ["prettyprint", "ffi", "chrono-tz"] }
63+
arrow-array = { version = "56.0.0" }
64+
arrow-cast = { version = "56.0.0" }
65+
arrow-data = { version = "56.0.0" }
66+
arrow-json = { version = "56.0.0" }
67+
arrow-schema = { version = "56.0.0" }
6868
async-trait = { version = "0.1.87" }
6969
bytes = "1.10"
7070
byteorder = "1"
7171
chrono = { version = "0.4.38", default-features = false }
72-
comfy-table = { version = "7.0" }
72+
comfy-table = { version = "7.1.4" }
7373
criterion = { version = "0.5", features = ["html_reports"] }
74-
datafusion = { version = "49.0.0", default-features = false }
75-
datafusion-catalog = { version = "49.0.0" }
76-
datafusion-common = { version = "49.0.0", default-features = false }
77-
datafusion-common-runtime = { version = "49.0.0", default-features = false }
78-
datafusion-execution = { version = "49.0.0", default-features = false }
79-
datafusion-expr = { version = "49.0.0" }
80-
datafusion-ffi = { version = "49.0.0" }
81-
datafusion-physical-expr = { version = "49.0.0" }
82-
datafusion-physical-plan = { version = "49.0.0" }
74+
datafusion = { version = "50.2.0", default-features = false }
75+
datafusion-catalog = { version = "50.2.0" }
76+
datafusion-common = { version = "50.2.0", default-features = false }
77+
datafusion-common-runtime = { version = "50.2.0", default-features = false }
78+
datafusion-datasource-parquet = { version = "50.2.0" }
79+
datafusion-execution = { version = "50.2.0", default-features = false }
80+
datafusion-expr = { version = "50.2.0" }
81+
datafusion-ffi = { version = "50.2.0" }
82+
datafusion-physical-expr = { version = "50.2.0" }
83+
datafusion-physical-plan = { version = "50.2.0" }
8384
dirs = "6.0.0"
8485
env_logger = "0.11"
8586
futures = { version = "0.3" }
@@ -102,7 +103,7 @@ wkb = "0.9.1"
102103
wkt = "0.14.0"
103104

104105
parking_lot = "0.12"
105-
parquet = { version = "55.1.0", default-features = false, features = [
106+
parquet = { version = "56.0.0", default-features = false, features = [
106107
"arrow",
107108
"async",
108109
"object_store",
@@ -119,16 +120,9 @@ tokio = { version = "1.44" }
119120
url = "2.5.4"
120121

121122
[patch.crates-io]
122-
# For R package build on Windows (can be removed when we upgrade to DataFusion main/51.0.0)
123-
datafusion = { git = "https://github.com/paleolimbot/datafusion.git", branch = "local-49-with-patch", package = "datafusion" }
124-
datafusion-catalog = { git = "https://github.com/paleolimbot/datafusion.git", branch = "local-49-with-patch", package = "datafusion-catalog" }
125-
datafusion-common = { git = "https://github.com/paleolimbot/datafusion.git", branch = "local-49-with-patch", package = "datafusion-common" }
126-
datafusion-common-runtime = { git = "https://github.com/paleolimbot/datafusion.git", branch = "local-49-with-patch", package = "datafusion-common-runtime" }
127-
datafusion-execution = { git = "https://github.com/paleolimbot/datafusion.git", branch = "local-49-with-patch", package = "datafusion-execution" }
128-
datafusion-expr = { git = "https://github.com/paleolimbot/datafusion.git", branch = "local-49-with-patch", package = "datafusion-expr" }
129-
datafusion-ffi = { git = "https://github.com/paleolimbot/datafusion.git", branch = "local-49-with-patch", package = "datafusion-ffi" }
130-
datafusion-physical-expr = { git = "https://github.com/paleolimbot/datafusion.git", branch = "local-49-with-patch", package = "datafusion-physical-expr" }
131-
datafusion-physical-plan = { git = "https://github.com/paleolimbot/datafusion.git", branch = "local-49-with-patch", package = "datafusion-physical-plan" }
123+
# Use main branch of arrow-adbc which supports Arrow 56.x (remove when 0.21.0 is released)
124+
adbc_core = { git = "https://github.com/apache/arrow-adbc.git", package = "adbc_core" }
125+
adbc_ffi = { git = "https://github.com/apache/arrow-adbc.git", package = "adbc_ffi" }
132126

133127
# TODO: remove them once changes we made to geo-index and wkb crates are merged to upstream and released
134128
geo-index = { git = "https://github.com/wherobots/geo-index.git", branch = "main" }

‎c/sedona-proj/src/st_transform.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -331,6 +331,7 @@ mod tests {
331331
use super::*;
332332
use arrow_array::ArrayRef;
333333
use arrow_schema::{DataType, Field};
334+
use datafusion_common::config::ConfigOptions;
334335
use datafusion_expr::{ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl};
335336
use rstest::rstest;
336337
use sedona_expr::scalar_udf::SedonaScalarUDF;
@@ -608,6 +609,7 @@ mod tests {
608609
arg_fields: arg_fields.to_vec(),
609610
number_rows: row_count,
610611
return_field,
612+
config_options: Arc::new(ConfigOptions::default()),
611613
};
612614

613615
let value = udf.invoke_with_args(args)?;

‎python/sedonadb/python/sedonadb/dataframe.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -441,7 +441,7 @@ def explain(
441441
│ utf8 ┆ utf8 │
442442
╞═══════════════╪═════════════════════════════════╡
443443
│ logical_plan ┆ Projection: Int64(1) AS one │
444-
│ ┆ EmptyRelation │
444+
│ ┆ EmptyRelation: rows=1 │
445445
├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤
446446
│ physical_plan ┆ ProjectionExec: expr=[1 as one] │
447447
│ ┆ PlaceholderRowExec │

‎python/sedonadb/tests/test_dataframe.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -444,7 +444,7 @@ def test_show_explained(con, capsys):
444444
│ utf8 ┆ utf8 │
445445
╞═══════════════╪═════════════════════════════════╡
446446
│ logical_plan ┆ Projection: Int64(1) AS one │
447-
│ ┆ EmptyRelation │
447+
│ ┆ EmptyRelation: rows=1 │
448448
├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤
449449
│ physical_plan ┆ ProjectionExec: expr=[1 as one] │
450450
│ ┆ PlaceholderRowExec │
@@ -462,7 +462,7 @@ def test_explain(con, capsys):
462462
│ utf8 ┆ utf8 │
463463
╞═══════════════╪═════════════════════════════════╡
464464
│ logical_plan ┆ Projection: Int64(1) AS one │
465-
│ ┆ EmptyRelation │
465+
│ ┆ EmptyRelation: rows=1 │
466466
├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤
467467
│ physical_plan ┆ ProjectionExec: expr=[1 as one] │
468468
│ ┆ PlaceholderRowExec │

‎rust/sedona-expr/src/aggregate_udf.rs‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,20 @@ pub struct SedonaAggregateUDF {
4141
documentation: Option<Documentation>,
4242
}
4343

44+
impl PartialEq for SedonaAggregateUDF {
45+
fn eq(&self, other: &Self) -> bool {
46+
self.name == other.name
47+
}
48+
}
49+
50+
impl Eq for SedonaAggregateUDF {}
51+
52+
impl std::hash::Hash for SedonaAggregateUDF {
53+
fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
54+
self.name.hash(state);
55+
}
56+
}
57+
4458
impl SedonaAggregateUDF {
4559
/// Create a new SedonaAggregateUDF
4660
pub fn new(

‎rust/sedona-expr/src/scalar_udf.rs‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,20 @@ pub struct SedonaScalarUDF {
4141
aliases: Vec<String>,
4242
}
4343

44+
impl PartialEq for SedonaScalarUDF {
45+
fn eq(&self, other: &Self) -> bool {
46+
self.name == other.name
47+
}
48+
}
49+
50+
impl Eq for SedonaScalarUDF {}
51+
52+
impl std::hash::Hash for SedonaScalarUDF {
53+
fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
54+
self.name.hash(state);
55+
}
56+
}
57+
4458
/// User-defined function implementation
4559
///
4660
/// A `SedonaScalarUdf` is comprised of one or more kernels, to which it dispatches

‎rust/sedona-expr/src/spatial_filter.rs‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -402,6 +402,7 @@ fn parse_args(args: &[Arc<dyn PhysicalExpr>]) -> Vec<ArgRef<'_>> {
402402
#[cfg(test)]
403403
mod test {
404404
use arrow_schema::{DataType, Field};
405+
use datafusion_common::config::ConfigOptions;
405406
use datafusion_expr::{ScalarUDF, Signature, SimpleScalarUDF, Volatility};
406407
use rstest::rstest;
407408
use sedona_geometry::{bounding_box::BoundingBox, interval::Interval};
@@ -589,6 +590,7 @@ mod test {
589590
Arc::new(unrelated),
590591
vec![],
591592
Arc::new(Field::new("", DataType::Boolean, true)),
593+
Arc::new(ConfigOptions::default()),
592594
));
593595
assert!(matches!(
594596
SpatialFilter::try_from_expr(&expr_no_args).unwrap(),
@@ -614,6 +616,7 @@ mod test {
614616
Arc::new(func.clone()),
615617
vec![column.clone(), literal.clone()],
616618
Arc::new(Field::new("", DataType::Boolean, true)),
619+
Arc::new(ConfigOptions::default()),
617620
));
618621
let predicate = SpatialFilter::try_from_expr(&expr).unwrap();
619622
assert!(
@@ -627,6 +630,7 @@ mod test {
627630
Arc::new(func),
628631
vec![literal.clone(), column.clone()],
629632
Arc::new(Field::new("", DataType::Boolean, true)),
633+
Arc::new(ConfigOptions::default()),
630634
));
631635
let predicate_reversed = SpatialFilter::try_from_expr(&expr_reversed).unwrap();
632636
assert!(
@@ -651,6 +655,7 @@ mod test {
651655
Arc::new(func.clone()),
652656
vec![column.clone(), literal.clone()],
653657
Arc::new(Field::new("", DataType::Boolean, true)),
658+
Arc::new(ConfigOptions::default()),
654659
));
655660
let predicate = SpatialFilter::try_from_expr(&expr).unwrap();
656661
assert!(
@@ -664,6 +669,7 @@ mod test {
664669
Arc::new(func),
665670
vec![literal.clone(), column.clone()],
666671
Arc::new(Field::new("", DataType::Boolean, true)),
672+
Arc::new(ConfigOptions::default()),
667673
));
668674
let predicate_reversed = SpatialFilter::try_from_expr(&expr_reversed).unwrap();
669675
assert!(
@@ -690,6 +696,7 @@ mod test {
690696
Arc::new(func.clone()),
691697
vec![column.clone(), literal.clone()],
692698
Arc::new(Field::new("", DataType::Boolean, true)),
699+
Arc::new(ConfigOptions::default()),
693700
));
694701
let predicate = SpatialFilter::try_from_expr(&expr).unwrap();
695702
assert!(
@@ -703,6 +710,7 @@ mod test {
703710
Arc::new(func),
704711
vec![literal.clone(), column.clone()],
705712
Arc::new(Field::new("", DataType::Boolean, true)),
713+
Arc::new(ConfigOptions::default()),
706714
));
707715
let predicate_reversed = SpatialFilter::try_from_expr(&expr_reversed).unwrap();
708716
assert!(
@@ -730,6 +738,7 @@ mod test {
730738
Arc::new(func.clone()),
731739
vec![column.clone(), literal.clone()],
732740
Arc::new(Field::new("", DataType::Boolean, true)),
741+
Arc::new(ConfigOptions::default()),
733742
));
734743
let predicate = SpatialFilter::try_from_expr(&expr).unwrap();
735744
assert!(
@@ -744,6 +753,7 @@ mod test {
744753
Arc::new(func),
745754
vec![literal.clone(), column.clone()],
746755
Arc::new(Field::new("", DataType::Boolean, true)),
756+
Arc::new(ConfigOptions::default()),
747757
));
748758
let predicate_reversed = SpatialFilter::try_from_expr(&expr_reversed).unwrap();
749759
assert!(
@@ -770,6 +780,7 @@ mod test {
770780
Arc::new(st_dwithin.clone()),
771781
vec![column.clone(), literal.clone(), distance_literal.clone()],
772782
Arc::new(Field::new("", DataType::Boolean, true)),
783+
Arc::new(ConfigOptions::default()),
773784
));
774785
let predicate = SpatialFilter::try_from_expr(&dwithin_expr).unwrap();
775786
assert!(
@@ -783,6 +794,7 @@ mod test {
783794
Arc::new(st_dwithin),
784795
vec![literal.clone(), column.clone(), distance_literal.clone()],
785796
Arc::new(Field::new("", DataType::Boolean, true)),
797+
Arc::new(ConfigOptions::default()),
786798
));
787799
let predicate_reversed = SpatialFilter::try_from_expr(&dwithin_expr_reversed).unwrap();
788800
assert!(
@@ -797,6 +809,7 @@ mod test {
797809
Arc::new(st_distance.clone()),
798810
vec![column.clone(), literal.clone()],
799811
Arc::new(Field::new("", DataType::Boolean, true)),
812+
Arc::new(ConfigOptions::default()),
800813
));
801814
let comparison_expr: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(
802815
distance_expr.clone(),
@@ -830,6 +843,7 @@ mod test {
830843
Arc::new(st_dwithin.clone()),
831844
vec![column.clone(), literal.clone(), negative_distance],
832845
Arc::new(Field::new("", DataType::Boolean, true)),
846+
Arc::new(ConfigOptions::default()),
833847
));
834848
let predicate = SpatialFilter::try_from_expr(&dwithin_expr).unwrap();
835849
assert!(
@@ -845,6 +859,7 @@ mod test {
845859
Arc::new(st_dwithin),
846860
vec![column.clone(), literal.clone(), nan_distance],
847861
Arc::new(Field::new("", DataType::Boolean, true)),
862+
Arc::new(ConfigOptions::default()),
848863
));
849864
let predicate_nan = SpatialFilter::try_from_expr(&dwithin_expr_nan).unwrap();
850865
assert!(
@@ -878,6 +893,7 @@ mod test {
878893
Arc::new(st_intersects.clone()),
879894
vec![],
880895
Arc::new(Field::new("", DataType::Boolean, true)),
896+
Arc::new(ConfigOptions::default()),
881897
));
882898
assert!(SpatialFilter::try_from_expr(&expr_no_args)
883899
.unwrap_err()
@@ -890,6 +906,7 @@ mod test {
890906
Arc::new(st_intersects.clone()),
891907
vec![literal.clone(), literal.clone()],
892908
Arc::new(Field::new("", DataType::Boolean, true)),
909+
Arc::new(ConfigOptions::default()),
893910
));
894911
assert!(matches!(
895912
SpatialFilter::try_from_expr(&expr_wrong_types).unwrap(),
@@ -926,6 +943,7 @@ mod test {
926943
Arc::new(func.clone()),
927944
vec![column.clone(), literal.clone()],
928945
Arc::new(Field::new("", DataType::Boolean, true)),
946+
Arc::new(ConfigOptions::default()),
929947
));
930948
let predicate = SpatialFilter::try_from_expr(&expr).unwrap();
931949
assert!(
@@ -952,6 +970,7 @@ mod test {
952970
Arc::new(st_dwithin.clone()),
953971
vec![column.clone(), literal.clone(), distance_literal.clone()],
954972
Arc::new(Field::new("", DataType::Boolean, true)),
973+
Arc::new(ConfigOptions::default()),
955974
));
956975
let predicate = SpatialFilter::try_from_expr(&dwithin_expr).unwrap();
957976
assert!(
@@ -965,6 +984,7 @@ mod test {
965984
Arc::new(st_dwithin),
966985
vec![literal.clone(), column.clone(), distance_literal.clone()],
967986
Arc::new(Field::new("", DataType::Boolean, true)),
987+
Arc::new(ConfigOptions::default()),
968988
));
969989
let predicate_reversed = SpatialFilter::try_from_expr(&dwithin_expr_reversed).unwrap();
970990
assert!(
@@ -979,6 +999,7 @@ mod test {
979999
Arc::new(st_distance.clone()),
9801000
vec![column.clone(), literal.clone()],
9811001
Arc::new(Field::new("", DataType::Boolean, true)),
1002+
Arc::new(ConfigOptions::default()),
9821003
));
9831004
let comparison_expr: Arc<dyn PhysicalExpr> = Arc::new(BinaryExpr::new(
9841005
distance_expr.clone(),
@@ -1014,6 +1035,7 @@ mod test {
10141035
Arc::new(has_z.clone()),
10151036
vec![column.clone()],
10161037
Arc::new(Field::new("", DataType::Boolean, true)),
1038+
Arc::new(ConfigOptions::default()),
10171039
));
10181040
let predicate = SpatialFilter::try_from_expr(&expr).unwrap();
10191041
assert!(matches!(predicate, SpatialFilter::HasZ(_)));
@@ -1029,6 +1051,7 @@ mod test {
10291051
Arc::new(has_z.clone()),
10301052
vec![],
10311053
Arc::new(Field::new("", DataType::Boolean, true)),
1054+
Arc::new(ConfigOptions::default()),
10321055
));
10331056
assert!(SpatialFilter::try_from_expr(&expr_no_args)
10341057
.unwrap_err()
@@ -1041,6 +1064,7 @@ mod test {
10411064
Arc::new(has_z.clone()),
10421065
vec![literal.clone()],
10431066
Arc::new(Field::new("", DataType::Boolean, true)),
1067+
Arc::new(ConfigOptions::default()),
10441068
));
10451069
assert!(matches!(
10461070
SpatialFilter::try_from_expr(&expr_wrong_types).unwrap(),

‎rust/sedona-functions/src/st_setsrid.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -232,6 +232,7 @@ mod test {
232232
use std::rc::Rc;
233233

234234
use arrow_schema::Field;
235+
use datafusion_common::config::ConfigOptions;
235236
use datafusion_expr::{ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF};
236237
use sedona_geometry::{error::SedonaGeometryError, transform::CrsTransform};
237238
use sedona_schema::{
@@ -374,6 +375,7 @@ mod test {
374375
arg_fields,
375376
number_rows: 1,
376377
return_field,
378+
config_options: Arc::new(ConfigOptions::default()),
377379
};
378380

379381
let value = udf.invoke_with_args(args)?;

‎rust/sedona-geoparquet/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ chrono = { workspace = true }
4646
datafusion = { workspace = true, features = ["parquet"] }
4747
datafusion-catalog = { workspace = true }
4848
datafusion-common = { workspace = true }
49+
datafusion-datasource-parquet = { workspace = true }
4950
datafusion-execution = { workspace = true }
5051
datafusion-expr = { workspace = true }
5152
datafusion-physical-expr = { workspace = true }

0 commit comments

Comments
 (0)