allow more customization in deltalake connector (#7478)

GitOrigin-RevId: 7bde44c2c966ec00aeb5de5bb4108f5970ce41cc
pathwaycom · Oct 15, 2024 · b4ca0a1 · b4ca0a1
1 parent 6178bf4
commit b4ca0a1
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 3 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 ## [Unreleased]
 
+### Added
+- `pw.io.deltalake.read` now supports custom S3 Delta Lakes with HTTP endpoints.
+- `pw.io.deltalake.read` now supports specifying both a custom endpoint and a custom region for Delta Lakes via `pw.io.s3.AwsS3Settings`.
+
 ## [0.15.1] - 2024-10-04
 
 ### Fixed

diff --git a/src/python_api.rs b/src/python_api.rs
@@ -4220,7 +4220,7 @@ impl DataStorage {
             storage_options.insert("AWS_PROFILE".to_string(), profile.to_string());
         }
 
-        if let s3::Region::Custom { endpoint, .. } = &s3_settings.region {
+        if let s3::Region::Custom { endpoint, region } = &s3_settings.region {
             if endpoint.starts_with("https://") || endpoint.starts_with("http://") {
                 storage_options.insert("AWS_ENDPOINT_URL".to_string(), endpoint.to_string());
             } else {
@@ -4229,12 +4229,15 @@ impl DataStorage {
                     format!("https://{endpoint}"),
                 );
             }
+            storage_options.insert("AWS_ALLOW_HTTP".to_string(), "True".to_string());
+            storage_options.insert("AWS_STORAGE_ALLOW_HTTP".to_string(), "True".to_string());
+            if region != endpoint {
+                storage_options.insert("AWS_REGION".to_string(), region.to_string());
+            }
         } else {
             storage_options.insert("AWS_REGION".to_string(), s3_settings.region.to_string());
         }
 
-        warn!("{:?}", storage_options);
-
         Ok(storage_options)
     }