From 85689625cb79fc037be67d44357b71a4411224fd Mon Sep 17 00:00:00 2001 From: JaySon-Huang Date: Tue, 28 Feb 2023 22:47:47 +0800 Subject: [PATCH] Speedup testing Signed-off-by: JaySon-Huang --- .../tests/gtest_s3_lock_service.cpp | 2 +- .../S3}/MockS3Client.cpp | 39 ++++++++++--------- .../{TestUtils => Storages/S3}/MockS3Client.h | 4 +- dbms/src/Storages/S3/S3Common.cpp | 10 ++++- dbms/src/Storages/S3/S3Common.h | 2 +- dbms/src/TestUtils/gtests_dbms_main.cpp | 1 + 6 files changed, 33 insertions(+), 25 deletions(-) rename dbms/src/{TestUtils => Storages/S3}/MockS3Client.cpp (66%) rename dbms/src/{TestUtils => Storages/S3}/MockS3Client.h (96%) diff --git a/dbms/src/Flash/Disaggregated/tests/gtest_s3_lock_service.cpp b/dbms/src/Flash/Disaggregated/tests/gtest_s3_lock_service.cpp index 9765771c9c0..8ea3b1db23c 100644 --- a/dbms/src/Flash/Disaggregated/tests/gtest_s3_lock_service.cpp +++ b/dbms/src/Flash/Disaggregated/tests/gtest_s3_lock_service.cpp @@ -15,10 +15,10 @@ #include #include #include +#include #include #include #include -#include #include #include #include diff --git a/dbms/src/TestUtils/MockS3Client.cpp b/dbms/src/Storages/S3/MockS3Client.cpp similarity index 66% rename from dbms/src/TestUtils/MockS3Client.cpp rename to dbms/src/Storages/S3/MockS3Client.cpp index cbeea2af3dc..d620a85e75c 100644 --- a/dbms/src/TestUtils/MockS3Client.cpp +++ b/dbms/src/Storages/S3/MockS3Client.cpp @@ -13,7 +13,7 @@ // limitations under the License. #include -#include +#include #include #include #include @@ -27,24 +27,25 @@ #include #include -namespace DB +namespace DB::S3::tests { +using namespace Aws::S3; -Aws::S3::Model::PutObjectOutcome MockS3Client::PutObject(const Aws::S3::Model::PutObjectRequest & r) const +Model::PutObjectOutcome MockS3Client::PutObject(const Model::PutObjectRequest & r) const { put_keys.emplace_back(r.GetKey()); - return Aws::S3::Model::PutObjectOutcome{Aws::AmazonWebServiceResult{}}; + return Model::PutObjectOutcome{Aws::AmazonWebServiceResult{}}; } -Aws::S3::Model::DeleteObjectOutcome MockS3Client::DeleteObject(const Aws::S3::Model::DeleteObjectRequest & r) const +Model::DeleteObjectOutcome MockS3Client::DeleteObject(const Model::DeleteObjectRequest & r) const { delete_keys.emplace_back(r.GetKey()); - return Aws::S3::Model::DeleteObjectOutcome{Aws::AmazonWebServiceResult{}}; + return Model::DeleteObjectOutcome{Aws::AmazonWebServiceResult{}}; } -Aws::S3::Model::ListObjectsV2Outcome MockS3Client::ListObjectsV2(const Aws::S3::Model::ListObjectsV2Request & r) const +Model::ListObjectsV2Outcome MockS3Client::ListObjectsV2(const Model::ListObjectsV2Request & r) const { - Aws::S3::Model::ListObjectsV2Result resp; + Model::ListObjectsV2Result resp; for (const auto & k : put_keys) { if (startsWith(k, r.GetPrefix())) @@ -60,7 +61,7 @@ Aws::S3::Model::ListObjectsV2Outcome MockS3Client::ListObjectsV2(const Aws::S3:: } if (is_deleted) continue; - Aws::S3::Model::Object o; + Model::Object o; o.SetKey(k); resp.AddContents(o); } @@ -80,33 +81,33 @@ Aws::S3::Model::ListObjectsV2Outcome MockS3Client::ListObjectsV2(const Aws::S3:: } if (is_deleted) continue; - Aws::S3::Model::Object o; + Model::Object o; o.SetKey(k); resp.AddContents(o); } } - return Aws::S3::Model::ListObjectsV2Outcome{resp}; + return Model::ListObjectsV2Outcome{resp}; } -Aws::S3::Model::HeadObjectOutcome MockS3Client::HeadObject(const Aws::S3::Model::HeadObjectRequest & r) const +Model::HeadObjectOutcome MockS3Client::HeadObject(const Model::HeadObjectRequest & r) const { for (const auto & k : put_keys) { if (r.GetKey() == k) { - Aws::S3::Model::HeadObjectResult resp; - return Aws::S3::Model::HeadObjectOutcome{resp}; + Model::HeadObjectResult resp; + return Model::HeadObjectOutcome{resp}; } } if (!head_result_mtime) { - Aws::Client::AWSError error(Aws::S3::S3Errors::NO_SUCH_KEY, false); - return Aws::S3::Model::HeadObjectOutcome{error}; + Aws::Client::AWSError error(S3Errors::NO_SUCH_KEY, false); + return Model::HeadObjectOutcome{error}; } - Aws::S3::Model::HeadObjectResult resp; + Model::HeadObjectResult resp; resp.SetLastModified(head_result_mtime.value()); - return Aws::S3::Model::HeadObjectOutcome{resp}; + return Model::HeadObjectOutcome{resp}; } void MockS3Client::clear() @@ -117,4 +118,4 @@ void MockS3Client::clear() head_result_mtime.reset(); } -} // namespace DB +} // namespace DB::S3::tests diff --git a/dbms/src/TestUtils/MockS3Client.h b/dbms/src/Storages/S3/MockS3Client.h similarity index 96% rename from dbms/src/TestUtils/MockS3Client.h rename to dbms/src/Storages/S3/MockS3Client.h index ab84e3e432c..5290dc69206 100644 --- a/dbms/src/TestUtils/MockS3Client.h +++ b/dbms/src/Storages/S3/MockS3Client.h @@ -18,7 +18,7 @@ #include #include -namespace DB +namespace DB::S3::tests { class MockS3Client final : public S3::TiFlashS3Client { @@ -43,4 +43,4 @@ class MockS3Client final : public S3::TiFlashS3Client std::optional head_result_mtime; Aws::S3::Model::HeadObjectOutcome HeadObject(const Aws::S3::Model::HeadObjectRequest & request) const override; }; -} // namespace DB +} // namespace DB::S3::tests diff --git a/dbms/src/Storages/S3/S3Common.cpp b/dbms/src/Storages/S3/S3Common.cpp index 5be888e4d03..2126e56af9f 100644 --- a/dbms/src/Storages/S3/S3Common.cpp +++ b/dbms/src/Storages/S3/S3Common.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -136,16 +137,21 @@ bool ClientFactory::isEnabled() const return config.isS3Enabled(); } -void ClientFactory::init(const StorageS3Config & config_) +void ClientFactory::init(const StorageS3Config & config_, bool mock_s3_) { config = config_; Aws::InitAPI(aws_options); Aws::Utils::Logging::InitializeAWSLogging(std::make_shared()); - shared_client = create(); + shared_client = mock_s3_ ? std::make_unique() : create(); + if (!mock_s3_) { auto raw_client = create(); shared_tiflash_client = std::make_shared(config.bucket, std::move(*raw_client.release())); } + else + { + shared_tiflash_client = std::make_shared(config.bucket, tests::MockS3Client()); + } } void ClientFactory::shutdown() diff --git a/dbms/src/Storages/S3/S3Common.h b/dbms/src/Storages/S3/S3Common.h index 4558b1bdc9e..121d0dd7af3 100644 --- a/dbms/src/Storages/S3/S3Common.h +++ b/dbms/src/Storages/S3/S3Common.h @@ -58,7 +58,7 @@ class ClientFactory bool isEnabled() const; - void init(const StorageS3Config & config_); + void init(const StorageS3Config & config_, bool mock_s3_ = false); void shutdown(); const String & bucket() const; diff --git a/dbms/src/TestUtils/gtests_dbms_main.cpp b/dbms/src/TestUtils/gtests_dbms_main.cpp index 961c76d5b58..2932b1ad36b 100644 --- a/dbms/src/TestUtils/gtests_dbms_main.cpp +++ b/dbms/src/TestUtils/gtests_dbms_main.cpp @@ -84,6 +84,7 @@ int main(int argc, char ** argv) .access_key_id = access_key_id, .secret_access_key = secret_access_key, }; + Poco::Environment::set("AWS_EC2_METADATA_DISABLED", "true"); // disable to speedup testing DB::S3::ClientFactory::instance().init(s3config); #ifdef FIU_ENABLE