Upgrade DataFusion to 19.0.0 (#691)

ballista-cli/Cargo.toml

@@ -33,8 +33,8 @@ ballista = { path = "../ballista/client", version = "0.11.0", features = [
     "standalone",
 ] }
 clap = { version = "3", features = ["derive", "cargo"] }
-datafusion = { git = "https://github.com/apache/arrow-datafusion", rev = "cfbb14d" }
-datafusion-cli = { git = "https://github.com/apache/arrow-datafusion", rev = "cfbb14d" }
+datafusion = "19.0.0"
+datafusion-cli = "19.0.0"
 dirs = "4.0.0"
 env_logger = "0.10"
 mimalloc = { version = "0.1", default-features = false }

ballista/client/Cargo.toml

@@ -31,8 +31,8 @@ rust-version = "1.63"
 ballista-core = { path = "../core", version = "0.11.0" }
 ballista-executor = { path = "../executor", version = "0.11.0", optional = true }
 ballista-scheduler = { path = "../scheduler", version = "0.11.0", optional = true }
-datafusion = { git = "https://github.com/apache/arrow-datafusion", rev = "cfbb14d" }
-datafusion-proto = { git = "https://github.com/apache/arrow-datafusion", rev = "cfbb14d" }
+datafusion = "19.0.0"
+datafusion-proto = "19.0.0"
 futures = "0.3"
 log = "0.4"
 parking_lot = "0.12"

ballista/client/README.md

@@ -85,7 +85,7 @@ To build a simple ballista example, add the following dependencies to your `Carg
 ```toml
 [dependencies]
 ballista = "0.10"
-datafusion = "14.0.0"
+datafusion = "19.0.0"
 tokio = "1.0"
 ```
 

ballista/core/Cargo.toml

@@ -46,13 +46,13 @@ simd = ["datafusion/simd"]
 [dependencies]
 ahash = { version = "0.8", default-features = false }
 
-arrow-flight = { version = "32.0.0", features = ["flight-sql-experimental"] }
+arrow-flight = { version = "33.0.0", features = ["flight-sql-experimental"] }
 async-trait = "0.1.41"
 chrono = { version = "0.4", default-features = false }
 clap = { version = "3", features = ["derive", "cargo"] }
-datafusion = { git = "https://github.com/apache/arrow-datafusion", rev = "cfbb14d" }
+datafusion = "19.0.0"
 datafusion-objectstore-hdfs = { version = "0.1.1", default-features = false, optional = true }
-datafusion-proto = { git = "https://github.com/apache/arrow-datafusion", rev = "cfbb14d" }
+datafusion-proto = "19.0.0"
 futures = "0.3"
 hashbrown = "0.13"
 
@@ -82,4 +82,7 @@ tempfile = "3"
 
 [build-dependencies]
 rustc_version = "0.4.0"
-tonic-build = { version = "0.8", default-features = false, features = ["transport", "prost"] }
+tonic-build = { version = "0.8", default-features = false, features = [
+    "transport",
+    "prost",
+] }

ballista/executor/Cargo.toml

@@ -38,28 +38,38 @@ default = ["mimalloc"]
 
 [dependencies]
 anyhow = "1"
-arrow = { version = "32.0.0" }
-arrow-flight = { version = "32.0.0" }
+arrow = { version = "33.0.0" }
+arrow-flight = { version = "33.0.0" }
 async-trait = "0.1.41"
 ballista-core = { path = "../core", version = "0.11.0" }
 chrono = { version = "0.4", default-features = false }
 configure_me = "0.4.0"
 dashmap = "5.4.0"
-datafusion = { git = "https://github.com/apache/arrow-datafusion", rev = "cfbb14d" }
-datafusion-proto = { git = "https://github.com/apache/arrow-datafusion", rev = "cfbb14d" }
+datafusion = "19.0.0"
+datafusion-proto = "19.0.0"
 futures = "0.3"
 hyper = "0.14.4"
 log = "0.4"
 mimalloc = { version = "0.1", default-features = false, optional = true }
 num_cpus = "1.13.0"
 parking_lot = "0.12"
 tempfile = "3"
-tokio = { version = "1.0", features = ["macros", "rt", "rt-multi-thread", "parking_lot", "signal"] }
+tokio = { version = "1.0", features = [
+    "macros",
+    "rt",
+    "rt-multi-thread",
+    "parking_lot",
+    "signal",
+] }
 tokio-stream = { version = "0.1", features = ["net"] }
 tonic = "0.8"
 tracing = "0.1.36"
 tracing-appender = "0.2.2"
-tracing-subscriber = { version = "0.3.15", features = ["fmt", "env-filter", "ansi"] }
+tracing-subscriber = { version = "0.3.15", features = [
+    "fmt",
+    "env-filter",
+    "ansi",
+] }
 uuid = { version = "1.0", features = ["v4"] }
 
 [dev-dependencies]

ballista/scheduler/Cargo.toml

@@ -43,16 +43,16 @@ sled = ["sled_package", "tokio-stream"]
 
 [dependencies]
 anyhow = "1"
-arrow-flight = { version = "32.0.0", features = ["flight-sql-experimental"] }
+arrow-flight = { version = "33.0.0", features = ["flight-sql-experimental"] }
 async-recursion = "1.0.0"
 async-trait = "0.1.41"
 ballista-core = { path = "../core", version = "0.11.0" }
 base64 = { version = "0.13", default-features = false }
 clap = { version = "3", features = ["derive", "cargo"] }
 configure_me = "0.4.0"
 dashmap = "5.4.0"
-datafusion = { git = "https://github.com/apache/arrow-datafusion", rev = "cfbb14d" }
-datafusion-proto = { git = "https://github.com/apache/arrow-datafusion", rev = "cfbb14d" }
+datafusion = "19.0.0"
+datafusion-proto = "19.0.0"
 etcd-client = { version = "0.10", optional = true }
 flatbuffers = { version = "22.9.29" }
 futures = "0.3"
@@ -78,7 +78,11 @@ tonic = "0.8"
 tower = { version = "0.4" }
 tracing = "0.1.36"
 tracing-appender = "0.2.2"
-tracing-subscriber = { version = "0.3.15", features = ["fmt", "env-filter", "ansi"] }
+tracing-subscriber = { version = "0.3.15", features = [
+    "fmt",
+    "env-filter",
+    "ansi",
+] }
 uuid = { version = "1.0", features = ["v4"] }
 warp = "0.3"
 
@@ -87,4 +91,7 @@ ballista-core = { path = "../core", version = "0.11.0" }
 
 [build-dependencies]
 configure_me_codegen = "=0.4.0"
-tonic-build = { version = "0.8", default-features = false, features = ["transport", "prost"] }
+tonic-build = { version = "0.8", default-features = false, features = [
+    "transport",
+    "prost",
+] }

benchmarks/Cargo.toml

@@ -34,8 +34,8 @@ snmalloc = ["snmalloc-rs"]
 
 [dependencies]
 ballista = { path = "../ballista/client", version = "0.11.0" }
-datafusion = { git = "https://github.com/apache/arrow-datafusion", rev = "cfbb14d" }
-datafusion-proto = { git = "https://github.com/apache/arrow-datafusion", rev = "cfbb14d" }
+datafusion = "19.0.0"
+datafusion-proto = "19.0.0"
 env_logger = "0.10"
 futures = "0.3"
 mimalloc = { version = "0.1", optional = true, default-features = false }
@@ -45,7 +45,12 @@ serde = "1.0.136"
 serde_json = "1.0.78"
 snmalloc-rs = { version = "0.3", optional = true }
 structopt = { version = "0.3", default-features = false }
-tokio = { version = "^1.0", features = ["macros", "rt", "rt-multi-thread", "parking_lot"] }
+tokio = { version = "^1.0", features = [
+    "macros",
+    "rt",
+    "rt-multi-thread",
+    "parking_lot",
+] }
 
 [dev-dependencies]
 ballista-core = { path = "../ballista/core", version = "0.11.0" }

examples/Cargo.toml

@@ -23,7 +23,7 @@ homepage = "https://github.com/apache/arrow-ballista"
 repository = "https://github.com/apache/arrow-ballista"
 authors = ["Apache Arrow <dev@arrow.apache.org>"]
 license = "Apache-2.0"
-keywords = [ "arrow", "distributed", "query", "sql" ]
+keywords = ["arrow", "distributed", "query", "sql"]
 edition = "2021"
 publish = false
 rust-version = "1.63"
@@ -35,9 +35,15 @@ required-features = ["ballista/standalone"]
 
 [dependencies]
 ballista = { path = "../ballista/client", version = "0.11.0" }
-datafusion = { git = "https://github.com/apache/arrow-datafusion", rev = "cfbb14d" }
+datafusion = "19.0.0"
 futures = "0.3"
 num_cpus = "1.13.0"
 prost = "0.11"
-tokio = { version = "1.0", features = ["macros", "rt", "rt-multi-thread", "sync", "parking_lot"] }
+tokio = { version = "1.0", features = [
+    "macros",
+    "rt",
+    "rt-multi-thread",
+    "sync",
+    "parking_lot",
+] }
 tonic = "0.8"