From 1bba52e3bf24c5b470d2f581826fefdeec563a1c Mon Sep 17 00:00:00 2001 From: CooooolFrog Date: Mon, 12 Dec 2022 20:52:05 +0800 Subject: [PATCH] chore: bump oss sdk (#475) * update oss client and adapt to the new configuration * add update log level return value * add default value for oss options --- Cargo.lock | 5 +++-- analytic_engine/src/setup.rs | 2 ++ analytic_engine/src/storage_options.rs | 18 +++++++++++++++++- common_util/src/config.rs | 2 +- components/object_store/Cargo.toml | 2 +- components/object_store/src/aliyun.rs | 14 +++++++++++--- server/src/http.rs | 2 +- 7 files changed, 36 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f5e0724e89..28a1b56179 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3746,8 +3746,9 @@ checksum = "3baf96e39c5359d2eb0dd6ccb42c62b91d9678aa68160d261b9e0ccbf9e9dea9" [[package]] name = "oss-rust-sdk" -version = "0.5.0" -source = "git+https://github.com/jiacai2050/oss-rust-sdk.git?rev=a6c54471d0b187667395d6cac931bd4e01a9f4e4#a6c54471d0b187667395d6cac931bd4e01a9f4e4" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24b457bcf7bb4b31a098c70f2ea83aace64937181ba6e88f9564bc4d05b655d5" dependencies = [ "async-trait", "base64 0.13.0", diff --git a/analytic_engine/src/setup.rs b/analytic_engine/src/setup.rs index 0b4f6aa972..8ab3aac552 100644 --- a/analytic_engine/src/setup.rs +++ b/analytic_engine/src/setup.rs @@ -393,6 +393,8 @@ fn open_storage( aliyun_opts.key_secret, aliyun_opts.endpoint, aliyun_opts.bucket, + aliyun_opts.pool_max_idle_per_host, + aliyun_opts.timeout, )); let oss_with_metrics = Arc::new(StoreWithMetrics::new(oss)); Arc::new( diff --git a/analytic_engine/src/storage_options.rs b/analytic_engine/src/storage_options.rs index d167b9e9dd..fd9f3af2aa 100644 --- a/analytic_engine/src/storage_options.rs +++ b/analytic_engine/src/storage_options.rs @@ -1,6 +1,8 @@ // Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. -use common_util::config::ReadableSize; +use std::time::Duration; + +use common_util::config::{ReadableDuration, ReadableSize}; use serde::Deserialize; #[derive(Debug, Clone, Deserialize)] @@ -54,4 +56,18 @@ pub struct AliyunOptions { pub endpoint: String, pub bucket: String, pub prefix: String, + #[serde(default = "AliyunOptions::default_pool_max_idle_per_host")] + pub pool_max_idle_per_host: usize, + #[serde(default = "AliyunOptions::default_timeout")] + pub timeout: ReadableDuration, +} + +impl AliyunOptions { + fn default_pool_max_idle_per_host() -> usize { + 1024 + } + + fn default_timeout() -> ReadableDuration { + ReadableDuration::from(Duration::from_secs(60)) + } } diff --git a/common_util/src/config.rs b/common_util/src/config.rs index 148b9ce06a..113e064e6d 100644 --- a/common_util/src/config.rs +++ b/common_util/src/config.rs @@ -291,7 +291,7 @@ impl<'de> Deserialize<'de> for ReadableSize { } } -#[derive(Clone, Copy, Debug, PartialEq, Eq, Ord, PartialOrd)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Ord, PartialOrd, Default)] pub struct ReadableDuration(pub Duration); impl Add for ReadableDuration { diff --git a/components/object_store/Cargo.toml b/components/object_store/Cargo.toml index 557843a5e9..b71f058d60 100644 --- a/components/object_store/Cargo.toml +++ b/components/object_store/Cargo.toml @@ -21,7 +21,7 @@ lazy_static = { workspace = true } log = { workspace = true } lru = { workspace = true } lru-weighted-cache = { git = "https://github.com/jiacai2050/lru-weighted-cache.git", rev = "1cf61aaf88469387e610dc7154fa318843491428" } -oss-rust-sdk = { git = "https://github.com/jiacai2050/oss-rust-sdk.git", rev = "a6c54471d0b187667395d6cac931bd4e01a9f4e4" } +oss-rust-sdk = "0.6.1" prost = { workspace = true } proto = { workspace = true } prometheus = { workspace = true } diff --git a/components/object_store/src/aliyun.rs b/components/object_store/src/aliyun.rs index 55a73a7653..49c6ca3421 100644 --- a/components/object_store/src/aliyun.rs +++ b/components/object_store/src/aliyun.rs @@ -1,6 +1,6 @@ // Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0. -use std::{collections::HashMap, fmt::Display, ops::Range}; +use std::{collections::HashMap, fmt::Display, ops::Range, time::Duration}; use async_trait::async_trait; use bytes::Bytes; @@ -8,7 +8,9 @@ use futures::{ stream::{self, BoxStream}, StreamExt, }; -use oss_rust_sdk::{async_object::AsyncObjectAPI, errors::Error as AliyunError, prelude::OSS}; +use oss_rust_sdk::{ + async_object::AsyncObjectAPI, errors::Error as AliyunError, oss::Options, prelude::OSS, +}; use snafu::{ResultExt, Snafu}; use tokio::io::AsyncWrite; use upstream::{ @@ -63,12 +65,18 @@ impl AliyunOSS { key_secret: impl Into, endpoint: impl Into, bucket: impl Into, + pool_max_idle_per_host: impl Into, + timeout: impl Into, ) -> Self { - let oss = OSS::new( + let oss = OSS::new_with_opts( key_id.into(), key_secret.into(), endpoint.into(), bucket.into(), + Options { + pool_max_idle_per_host: Some(pool_max_idle_per_host.into()), + timeout: Some(timeout.into()), + }, ); Self { oss } } diff --git a/server/src/http.rs b/server/src/http.rs index ca715f06f7..2c283a2ef9 100644 --- a/server/src/http.rs +++ b/server/src/http.rs @@ -252,7 +252,7 @@ impl Service { .set_level_by_str(log_level.as_str()) .map_err(|e| Error::HandleUpdateLogLevel { msg: e }); match result { - Ok(()) => Ok(reply::reply()), + Ok(()) => Ok(reply::json(&log_level)), Err(e) => Err(reject::custom(e)), } },