From 10e34258774d5a803049221fd2554e056995317c Mon Sep 17 00:00:00 2001 From: QP Hou Date: Fri, 7 May 2021 11:49:45 -0700 Subject: [PATCH] Add back azure support after the crates release (#239) This reverts commit 4ad7426a8d7cb05006a31a81285f6ad6d9ea9c87. --- .github/workflows/build.yml | 4 +- Cargo.lock | 306 ++++++++++++++++++++++++++++++++++++ python/Cargo.toml | 2 +- rust/Cargo.toml | 9 +- 4 files changed, 313 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7e8cfeb6df..1511962370 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -42,7 +42,7 @@ jobs: toolchain: stable override: true - name: build and lint with clippy - run: cargo clippy --features datafusion-ext,s3,dynamodb + run: cargo clippy --features azure,datafusion-ext,s3,dynamodb test: strategy: @@ -62,7 +62,7 @@ jobs: toolchain: stable override: true - name: Run tests - run: cargo test --verbose --features datafusion-ext + run: cargo test --verbose --features datafusion-ext,azure s3_test: runs-on: ubuntu-latest diff --git a/Cargo.lock b/Cargo.lock index 5f337452ef..8e991af0d0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,21 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "RustyXML" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b5ace29ee3216de37c0546865ad08edef58b0f9e76838ed8959a84a990e58c5" + +[[package]] +name = "addr2line" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a55f82cfe485775d02112886f4169bde0c5894d75e79ead7eafe7e40a25e45f7" +dependencies = [ + "gimli", +] + [[package]] name = "adler" version = "1.0.2" @@ -130,6 +145,80 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" +[[package]] +name = "azure_core" +version = "0.1.0" +source = "git+https://github.com/Azure/azure-sdk-for-rust?rev=536da42ebefd411feff8ba6a0965865e2741267e#536da42ebefd411feff8ba6a0965865e2741267e" +dependencies = [ + "RustyXML", + "async-trait", + "base64 0.13.0", + "bytes", + "chrono", + "failure", + "futures", + "http", + "hyper", + "hyper-rustls", + "log", + "md5", + "oauth2", + "paste 1.0.5", + "quick-error", + "reqwest", + "serde", + "serde-xml-rs", + "serde_derive", + "serde_json", + "url", + "uuid", +] + +[[package]] +name = "azure_storage" +version = "0.1.0" +source = "git+https://github.com/Azure/azure-sdk-for-rust?rev=536da42ebefd411feff8ba6a0965865e2741267e#536da42ebefd411feff8ba6a0965865e2741267e" +dependencies = [ + "RustyXML", + "azure_core", + "base64 0.13.0", + "bytes", + "chrono", + "failure", + "futures", + "http", + "hyper", + "hyper-rustls", + "log", + "md5", + "mime", + "percent-encoding", + "quick-error", + "ring", + "serde", + "serde-xml-rs", + "serde_derive", + "serde_json", + "smallvec", + "time 0.2.26", + "url", + "uuid", +] + +[[package]] +name = "backtrace" +version = "0.3.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d117600f438b1707d4e4ae15d3595657288f8235a0eb593e80ecc98ab34e1bc" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + [[package]] name = "base-x" version = "0.2.8" @@ -451,6 +540,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "ct-logs" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1a816186fa68d9e426e3cb4ae4dff1fcd8e4a2c34b781bf7a822574a0d0aac8" +dependencies = [ + "sct", +] + [[package]] name = "ctor" version = "0.1.20" @@ -499,6 +597,8 @@ dependencies = [ "anyhow", "arrow", "async-trait", + "azure_core", + "azure_storage", "bytes", "cfg-if", "chrono", @@ -516,6 +616,7 @@ dependencies = [ "parquet", "pretty_assertions", "regex", + "reqwest", "rusoto_core", "rusoto_credential", "rusoto_dynamodb", @@ -663,6 +764,28 @@ dependencies = [ "libc", ] +[[package]] +name = "failure" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86" +dependencies = [ + "backtrace", + "failure_derive", +] + +[[package]] +name = "failure_derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "flatbuffers" version = "0.8.4" @@ -874,6 +997,12 @@ dependencies = [ "syn", ] +[[package]] +name = "gimli" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6503fe142514ca4799d4c26297c4248239fe8838d827db6bd6065c6ed29a6ce" + [[package]] name = "glibc_version" version = "0.1.2" @@ -1022,6 +1151,23 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f9f7a97316d44c0af9b0301e65010573a853a9fc97046d7331d7f6bc0fd5a64" +dependencies = [ + "ct-logs", + "futures-util", + "hyper", + "log", + "rustls", + "rustls-native-certs", + "tokio", + "tokio-rustls", + "webpki", +] + [[package]] name = "hyper-tls" version = "0.5.0" @@ -1416,6 +1562,30 @@ dependencies = [ "libc", ] +[[package]] +name = "oauth2" +version = "4.0.0-alpha.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14be0cf9da49c1a5b2909ec626949dc549e0adda4e9e7f26e4908f08a5435227" +dependencies = [ + "base64 0.12.3", + "chrono", + "http", + "rand 0.7.3", + "reqwest", + "serde", + "serde_json", + "sha2", + "thiserror", + "url", +] + +[[package]] +name = "object" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9a7ab5d64814df0fe4a4b5ead45ed6c5f181ee3ff04ba344313a6c80446c5d4" + [[package]] name = "once_cell" version = "1.7.2" @@ -1743,6 +1913,12 @@ dependencies = [ "syn", ] +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + [[package]] name = "quote" version = "1.0.9" @@ -1955,6 +2131,7 @@ dependencies = [ "http", "http-body", "hyper", + "hyper-rustls", "hyper-tls", "ipnet", "js-sys", @@ -1964,17 +2141,35 @@ dependencies = [ "native-tls", "percent-encoding", "pin-project-lite", + "rustls", "serde", "serde_urlencoded 0.7.0", "tokio", "tokio-native-tls", + "tokio-rustls", "url", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", + "webpki-roots", "winreg", ] +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi", +] + [[package]] name = "rusoto_core" version = "0.46.0" @@ -2097,6 +2292,12 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "rustc-demangle" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e3bad0ee36814ca07d7968269dd4b7ec89ec2da10c4bb613928d3077083c232" + [[package]] name = "rustc_version" version = "0.2.3" @@ -2106,6 +2307,31 @@ dependencies = [ "semver", ] +[[package]] +name = "rustls" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7" +dependencies = [ + "base64 0.13.0", + "log", + "ring", + "sct", + "webpki", +] + +[[package]] +name = "rustls-native-certs" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a07b7c1885bd8ed3831c289b7870b13ef46fe0e856d288c30d9cc17d75a2092" +dependencies = [ + "openssl-probe", + "rustls", + "schannel", + "security-framework", +] + [[package]] name = "rustyline" version = "7.1.0" @@ -2159,6 +2385,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "sct" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "security-framework" version = "2.2.0" @@ -2206,6 +2442,18 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-xml-rs" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0bf1ba0696ccf0872866277143ff1fd14d22eec235d2b23702f95e6660f7dfa" +dependencies = [ + "log", + "serde", + "thiserror", + "xml-rs", +] + [[package]] name = "serde_derive" version = "1.0.125" @@ -2320,6 +2568,9 @@ name = "smallvec" version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" +dependencies = [ + "serde", +] [[package]] name = "snap" @@ -2337,6 +2588,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + [[package]] name = "sqlparser" version = "0.9.0" @@ -2439,6 +2696,18 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "synstructure" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "unicode-xid", +] + [[package]] name = "tempdir" version = "0.3.7" @@ -2646,6 +2915,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" +dependencies = [ + "rustls", + "tokio", + "webpki", +] + [[package]] name = "tokio-stream" version = "0.1.5" @@ -2751,6 +3031,12 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f14ee04d9415b52b3aeab06258a3f07093182b88ba0f9b8d203f211a7a7d41c7" +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + [[package]] name = "url" version = "2.2.1" @@ -2761,6 +3047,7 @@ dependencies = [ "idna", "matches", "percent-encoding", + "serde", ] [[package]] @@ -2907,6 +3194,25 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki" +version = "0.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "webpki-roots" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aabe153544e473b775453675851ecc86863d2a81d786d741f6b76778f2a48940" +dependencies = [ + "webpki", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/python/Cargo.toml b/python/Cargo.toml index b586ba82d1..b0e136ae7f 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -28,7 +28,7 @@ features = ["extension-module", "abi3", "abi3-py36"] [dependencies.deltalake] path = "../rust" version = "0" -features = ["s3"] +features = ["s3", "azure"] [package.metadata.maturin] name = "deltalake" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index cc4488737d..df80490756 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -28,9 +28,9 @@ uuid = { version = "0.8", features = ["serde", "v4"] } lazy_static = "1" # Azure -# reqwest = { version = "0", optional = true } -# azure_core = { git = "https://github.com/Azure/azure-sdk-for-rust", optional = true, rev = "536da42ebefd411feff8ba6a0965865e2741267e" } -# azure_storage = { git = "https://github.com/Azure/azure-sdk-for-rust", optional = true, rev = "536da42ebefd411feff8ba6a0965865e2741267e", features = ["blob", "account", "adls_gen2"] } +reqwest = { version = "0", optional = true } +azure_core = { git = "https://github.com/Azure/azure-sdk-for-rust", optional = true, rev = "536da42ebefd411feff8ba6a0965865e2741267e" } +azure_storage = { git = "https://github.com/Azure/azure-sdk-for-rust", optional = true, rev = "536da42ebefd411feff8ba6a0965865e2741267e", features = ["blob", "account", "adls_gen2"] } # S3 rusoto_core = { version = "0.46", optional = true } @@ -53,8 +53,7 @@ async-trait = "0.1" [features] rust-dataframe-ext = [] datafusion-ext = ["datafusion", "crossbeam"] -# azure = ["azure_core", "azure_storage", "reqwest"] -azure = [] +azure = ["azure_core", "azure_storage", "reqwest"] s3 = ["rusoto_core", "rusoto_credential", "rusoto_s3", "rusoto_sts"] dynamodb = ["rusoto_dynamodb", "maplit", "s3"]