feat: partitioned writes and data skipping for Delta Lake

Cargo.toml

@@ -119,6 +119,7 @@ datafusion-common = { version = "48.0.1", features = ["object_store", "avro"] }
 datafusion-expr = { version = "48.0.1" }
 datafusion-expr-common = { version = "48.0.1" }
 datafusion-execution = { version = "48.0.1" }
+datafusion-physical-optimizer = { version = "48.0.1" }
 datafusion-proto = { version = "48.0.1" }
 datafusion-functions = { version = "48.0.1" }
 datafusion-functions-nested = { version = "48.0.1" }
@@ -140,7 +141,7 @@ object_store = { version = "0.12.3", features = ["aws", "gcp", "azure", "http"]
 hdfs-native-object-store = "0.14.2"
 
 # Lakehouse
-deltalake = { git = "https://github.com/delta-io/delta-rs.git", rev = "3607c31", default-features = false, features = ["rustls"] }
+deltalake = { git = "https://github.com/delta-io/delta-rs.git", rev = "9a2c8ea", default-features = false, features = ["rustls"] }
 delta_kernel = { version = "0.13.0", features = ["arrow-55", "internal-api", "default-engine"] }
 parquet = { version = "55.2.0" }
 bytes = "1.9.0"

crates/sail-data-source/src/formats/delta.rs

@@ -45,8 +45,7 @@ impl TableFormat for DeltaTableFormat {
             input,
             path,
             mode,
-            // TODO: support partitioning
-            partition_by: _,
+            partition_by,
             bucket_by,
             sort_order,
             options,
@@ -65,7 +64,13 @@ impl TableFormat for DeltaTableFormat {
                 return not_impl_err!("unsupported sink mode for Delta: {mode:?}")
             }
         };
-        let sink = Arc::new(DeltaDataSink::new(mode, table_url, options, input.schema()));
+        let sink = Arc::new(DeltaDataSink::new(
+            mode,
+            table_url,
+            options,
+            input.schema(),
+            partition_by,
+        ));
 
         Ok(Arc::new(DataSinkExec::new(input, sink, sort_order)))
     }

crates/sail-delta-lake/Cargo.toml

@@ -19,6 +19,7 @@ datafusion = { workspace = true }
 datafusion-common = { workspace = true }
 datafusion-expr = { workspace = true }
 datafusion-execution = { workspace = true }
+datafusion-physical-optimizer = { workspace = true }
 
 # Arrow dependencies
 arrow-buffer = { workspace = true }
@@ -41,3 +42,7 @@ parquet = { workspace = true }
 bytes = { workspace = true }
 indexmap = { workspace = true }
 percent-encoding = { workspace = true }
+log = { workspace = true }
+
+[lints]
+workspace = true