From 364fc25766ca166e7ebcef02c764ce1e32a372ec Mon Sep 17 00:00:00 2001 From: Cai Yudong Date: Sat, 18 Jan 2020 10:07:46 +0800 Subject: [PATCH] rename 'minio' to 's3' in Storage Config section (#1088) * #1002 rename 'minio' to 's3' in Storage Config section * #1002 clang format --- CHANGELOG.md | 1 + core/conf/demo/server_config.yaml | 24 ++-- core/conf/server_cpu_config.template | 24 ++-- core/conf/server_gpu_config.template | 24 ++-- core/src/db/Utils.cpp | 12 +- core/src/server/Config.cpp | 133 +++++++++++------------ core/src/server/Config.h | 60 +++++----- core/src/storage/s3/S3ClientWrapper.cpp | 48 ++++---- core/src/storage/s3/S3ClientWrapper.h | 10 +- core/src/wrapper/VecIndex.cpp | 12 +- core/unittest/server/test_config.cpp | 78 ++++++------- core/unittest/storage/test_s3_client.cpp | 4 +- core/unittest/storage/utils.cpp | 12 +- 13 files changed, 221 insertions(+), 221 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 518dbd3bb7381..bea2a15ac1c7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ Please mark all change in change log and use the issue from GitHub - \#908 - Move "primary_path" and "secondary_path" to storage config - \#931 - Remove "collector" from config - \#966 - Update NOTICE.md +- \#1002 - Rename minio to s3 in Storage Config section ## Task diff --git a/core/conf/demo/server_config.yaml b/core/conf/demo/server_config.yaml index 60110444fe139..44704226dfa5a 100644 --- a/core/conf/demo/server_config.yaml +++ b/core/conf/demo/server_config.yaml @@ -70,27 +70,27 @@ db_config: # secondary_path | A semicolon-separated list of secondary directories used | Path | | # | to save vector data and index data. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ -# minio_enable | Enable MinIO storage or not. | Boolean | false | +# s3_enable | Enable Simple Storage Service or not. | Boolean | false | #----------------------+------------------------------------------------------------+------------+-----------------+ -# minio_address | MinIO storage service IP address. | IP | 127.0.0.1 | +# s3_address | Simple Storage Service IP address. | IP | 127.0.0.1 | #----------------------+------------------------------------------------------------+------------+-----------------+ -# minio_port | MinIO storage service port. Port range (1024, 65535) | Integer | 9000 | +# s3_port | Simple Storage Service port. Port range (1024, 65535) | Integer | 9000 | #----------------------+------------------------------------------------------------+------------+-----------------+ -# minio_access_key | MinIO storage service access key. | String | minioadmin | +# s3_access_key | Simple Storage Service access key. | String | minioadmin | #----------------------+------------------------------------------------------------+------------+-----------------+ -# minio_secret_key | MinIO storage service secret key. | String | minioadmin | +# s3_secret_key | Simple Storage Service secret key. | String | minioadmin | #----------------------+------------------------------------------------------------+------------+-----------------+ -# minio_bucket | MinIO storage service bucket name. | String | milvus-bucket | +# s3_bucket | Simple Storage Service bucket name. | String | milvus-bucket | #----------------------+------------------------------------------------------------+------------+-----------------+ storage_config: primary_path: /var/lib/milvus secondary_path: - minio_enable: false - minio_address: 127.0.0.1 - minio_port: 9000 - minio_access_key: minioadmin - minio_secret_key: minioadmin - minio_bucket: milvus-bucket + s3_enable: false + s3_address: 127.0.0.1 + s3_port: 9000 + s3_access_key: minioadmin + s3_secret_key: minioadmin + s3_bucket: milvus-bucket #----------------------+------------------------------------------------------------+------------+-----------------+ # Metric Config | Description | Type | Default | diff --git a/core/conf/server_cpu_config.template b/core/conf/server_cpu_config.template index b31cf5cb27a50..1e9866515f549 100644 --- a/core/conf/server_cpu_config.template +++ b/core/conf/server_cpu_config.template @@ -70,27 +70,27 @@ db_config: # secondary_path | A semicolon-separated list of secondary directories used | Path | | # | to save vector data and index data. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ -# minio_enable | Enable MinIO storage or not. | Boolean | false | +# s3_enable | Enable Simple Storage Service or not. | Boolean | false | #----------------------+------------------------------------------------------------+------------+-----------------+ -# minio_address | MinIO storage service IP address. | IP | 127.0.0.1 | +# s3_address | Simple Storage Service IP address. | IP | 127.0.0.1 | #----------------------+------------------------------------------------------------+------------+-----------------+ -# minio_port | MinIO storage service port. Port range (1024, 65535) | Integer | 9000 | +# s3_port | Simple Storage Service port. Port range (1024, 65535) | Integer | 9000 | #----------------------+------------------------------------------------------------+------------+-----------------+ -# minio_access_key | MinIO storage service access key. | String | minioadmin | +# s3_access_key | Simple Storage Service access key. | String | minioadmin | #----------------------+------------------------------------------------------------+------------+-----------------+ -# minio_secret_key | MinIO storage service secret key. | String | minioadmin | +# s3_secret_key | Simple Storage Service secret key. | String | minioadmin | #----------------------+------------------------------------------------------------+------------+-----------------+ -# minio_bucket | MinIO storage service bucket name. | String | milvus-bucket | +# s3_bucket | Simple Storage Service bucket name. | String | milvus-bucket | #----------------------+------------------------------------------------------------+------------+-----------------+ storage_config: primary_path: @MILVUS_DB_PATH@ secondary_path: - minio_enable: false - minio_address: 127.0.0.1 - minio_port: 9000 - minio_access_key: minioadmin - minio_secret_key: minioadmin - minio_bucket: milvus-bucket + s3_enable: false + s3_address: 127.0.0.1 + s3_port: 9000 + s3_access_key: minioadmin + s3_secret_key: minioadmin + s3_bucket: milvus-bucket #----------------------+------------------------------------------------------------+------------+-----------------+ # Metric Config | Description | Type | Default | diff --git a/core/conf/server_gpu_config.template b/core/conf/server_gpu_config.template index cbaa6be03b0d1..93d4f112d973f 100644 --- a/core/conf/server_gpu_config.template +++ b/core/conf/server_gpu_config.template @@ -70,27 +70,27 @@ db_config: # secondary_path | A semicolon-separated list of secondary directories used | Path | | # | to save vector data and index data. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ -# minio_enable | Enable MinIO storage or not. | Boolean | false | +# s3_enable | Enable Simple Storage Service or not. | Boolean | false | #----------------------+------------------------------------------------------------+------------+-----------------+ -# minio_address | MinIO storage service IP address. | IP | 127.0.0.1 | +# s3_address | Simple Storage Service IP address. | IP | 127.0.0.1 | #----------------------+------------------------------------------------------------+------------+-----------------+ -# minio_port | MinIO storage service port. Port range (1024, 65535) | Integer | 9000 | +# s3_port | Simple Storage Service port. Port range (1024, 65535) | Integer | 9000 | #----------------------+------------------------------------------------------------+------------+-----------------+ -# minio_access_key | MinIO storage service access key. | String | minioadmin | +# s3_access_key | Simple Storage Service access key. | String | minioadmin | #----------------------+------------------------------------------------------------+------------+-----------------+ -# minio_secret_key | MinIO storage service secret key. | String | minioadmin | +# s3_secret_key | Simple Storage Service secret key. | String | minioadmin | #----------------------+------------------------------------------------------------+------------+-----------------+ -# minio_bucket | MinIO storage service bucket name. | String | milvus-bucket | +# s3_bucket | Simple Storage Service bucket name. | String | milvus-bucket | #----------------------+------------------------------------------------------------+------------+-----------------+ storage_config: primary_path: @MILVUS_DB_PATH@ secondary_path: - minio_enable: false - minio_address: 127.0.0.1 - minio_port: 9000 - minio_access_key: minioadmin - minio_secret_key: minioadmin - minio_bucket: milvus-bucket + s3_enable: false + s3_address: 127.0.0.1 + s3_port: 9000 + s3_access_key: minioadmin + s3_secret_key: minioadmin + s3_bucket: milvus-bucket #----------------------+------------------------------------------------------------+------------+-----------------+ # Metric Config | Description | Type | Default | diff --git a/core/src/db/Utils.cpp b/core/src/db/Utils.cpp index fd03043f3f999..80152459bd05c 100644 --- a/core/src/db/Utils.cpp +++ b/core/src/db/Utils.cpp @@ -119,11 +119,11 @@ DeleteTablePath(const DBMetaOptions& options, const std::string& table_id, bool } } - bool minio_enable = false; + bool s3_enable = false; server::Config& config = server::Config::GetInstance(); - config.GetStorageConfigMinioEnable(minio_enable); + config.GetStorageConfigS3Enable(s3_enable); - if (minio_enable) { + if (s3_enable) { std::string table_path = options.path_ + TABLES_FOLDER + table_id; auto& storage_inst = milvus::storage::S3ClientWrapper::GetInstance(); @@ -156,10 +156,10 @@ GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file std::string parent_path = ConstructParentFolder(options.path_, table_file); std::string file_path = parent_path + "/" + table_file.file_id_; - bool minio_enable = false; + bool s3_enable = false; server::Config& config = server::Config::GetInstance(); - config.GetStorageConfigMinioEnable(minio_enable); - if (minio_enable) { + config.GetStorageConfigS3Enable(s3_enable); + if (s3_enable) { /* need not check file existence */ table_file.location_ = file_path; return Status::OK(); diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index ece734db2e057..ca2a9442b749e 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -111,24 +111,24 @@ Config::ValidateConfig() { std::string storage_secondary_path; CONFIG_CHECK(GetStorageConfigSecondaryPath(storage_secondary_path)); - bool storage_minio_enable; - CONFIG_CHECK(GetStorageConfigMinioEnable(storage_minio_enable)); - std::cout << "MinIO " << (storage_minio_enable ? "ENABLED !" : "DISABLED !") << std::endl; + bool storage_s3_enable; + CONFIG_CHECK(GetStorageConfigS3Enable(storage_s3_enable)); + std::cout << "S3 " << (storage_s3_enable ? "ENABLED !" : "DISABLED !") << std::endl; - std::string storage_minio_address; - CONFIG_CHECK(GetStorageConfigMinioAddress(storage_minio_address)); + std::string storage_s3_address; + CONFIG_CHECK(GetStorageConfigS3Address(storage_s3_address)); - std::string storage_minio_port; - CONFIG_CHECK(GetStorageConfigMinioPort(storage_minio_port)); + std::string storage_s3_port; + CONFIG_CHECK(GetStorageConfigS3Port(storage_s3_port)); - std::string storage_minio_access_key; - CONFIG_CHECK(GetStorageConfigMinioAccessKey(storage_minio_access_key)); + std::string storage_s3_access_key; + CONFIG_CHECK(GetStorageConfigS3AccessKey(storage_s3_access_key)); - std::string storage_minio_secret_key; - CONFIG_CHECK(GetStorageConfigMinioSecretKey(storage_minio_secret_key)); + std::string storage_s3_secret_key; + CONFIG_CHECK(GetStorageConfigS3SecretKey(storage_s3_secret_key)); - std::string storage_minio_bucket; - CONFIG_CHECK(GetStorageConfigMinioBucket(storage_minio_bucket)); + std::string storage_s3_bucket; + CONFIG_CHECK(GetStorageConfigS3Bucket(storage_s3_bucket)); /* metric config */ bool metric_enable_monitor; @@ -208,12 +208,12 @@ Config::ResetDefaultConfig() { /* storage config */ CONFIG_CHECK(SetStorageConfigPrimaryPath(CONFIG_STORAGE_PRIMARY_PATH_DEFAULT)); CONFIG_CHECK(SetStorageConfigSecondaryPath(CONFIG_STORAGE_SECONDARY_PATH_DEFAULT)); - CONFIG_CHECK(SetStorageConfigMinioEnable(CONFIG_STORAGE_MINIO_ENABLE_DEFAULT)); - CONFIG_CHECK(SetStorageConfigMinioAddress(CONFIG_STORAGE_MINIO_ADDRESS_DEFAULT)); - CONFIG_CHECK(SetStorageConfigMinioPort(CONFIG_STORAGE_MINIO_PORT_DEFAULT)); - CONFIG_CHECK(SetStorageConfigMinioAccessKey(CONFIG_STORAGE_MINIO_ACCESS_KEY_DEFAULT)); - CONFIG_CHECK(SetStorageConfigMinioSecretKey(CONFIG_STORAGE_MINIO_SECRET_KEY_DEFAULT)); - CONFIG_CHECK(SetStorageConfigMinioBucket(CONFIG_STORAGE_MINIO_BUCKET_DEFAULT)); + CONFIG_CHECK(SetStorageConfigS3Enable(CONFIG_STORAGE_S3_ENABLE_DEFAULT)); + CONFIG_CHECK(SetStorageConfigS3Address(CONFIG_STORAGE_S3_ADDRESS_DEFAULT)); + CONFIG_CHECK(SetStorageConfigS3Port(CONFIG_STORAGE_S3_PORT_DEFAULT)); + CONFIG_CHECK(SetStorageConfigS3AccessKey(CONFIG_STORAGE_S3_ACCESS_KEY_DEFAULT)); + CONFIG_CHECK(SetStorageConfigS3SecretKey(CONFIG_STORAGE_S3_SECRET_KEY_DEFAULT)); + CONFIG_CHECK(SetStorageConfigS3Bucket(CONFIG_STORAGE_S3_BUCKET_DEFAULT)); /* metric config */ CONFIG_CHECK(SetMetricConfigEnableMonitor(CONFIG_METRIC_ENABLE_MONITOR_DEFAULT)); @@ -497,35 +497,34 @@ Config::CheckStorageConfigSecondaryPath(const std::string& value) { } Status -Config::CheckStorageConfigMinioEnable(const std::string& value) { +Config::CheckStorageConfigS3Enable(const std::string& value) { if (!ValidationUtil::ValidateStringIsBool(value).ok()) { std::string msg = - "Invalid storage config: " + value + ". Possible reason: storage_config.minio_enable is not a boolean."; + "Invalid storage config: " + value + ". Possible reason: storage_config.s3_enable is not a boolean."; return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); } Status -Config::CheckStorageConfigMinioAddress(const std::string& value) { +Config::CheckStorageConfigS3Address(const std::string& value) { if (!ValidationUtil::ValidateIpAddress(value).ok()) { - std::string msg = - "Invalid minio address: " + value + ". Possible reason: storage_config.minio_address is invalid."; + std::string msg = "Invalid s3 address: " + value + ". Possible reason: storage_config.s3_address is invalid."; return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); } Status -Config::CheckStorageConfigMinioPort(const std::string& value) { +Config::CheckStorageConfigS3Port(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { - std::string msg = "Invalid minio port: " + value + ". Possible reason: storage_config.port is not a number."; + std::string msg = "Invalid s3 port: " + value + ". Possible reason: storage_config.s3_port is not a number."; return Status(SERVER_INVALID_ARGUMENT, msg); } else { int32_t port = std::stoi(value); if (!(port > 1024 && port < 65535)) { - std::string msg = "Invalid minio port: " + value + - ". Possible reason: storage_config.port is not in range (1024, 65535)."; + std::string msg = "Invalid s3 port: " + value + + ". Possible reason: storage_config.s3_port is not in range (1024, 65535)."; return Status(SERVER_INVALID_ARGUMENT, msg); } } @@ -533,25 +532,25 @@ Config::CheckStorageConfigMinioPort(const std::string& value) { } Status -Config::CheckStorageConfigMinioAccessKey(const std::string& value) { +Config::CheckStorageConfigS3AccessKey(const std::string& value) { if (value.empty()) { - return Status(SERVER_INVALID_ARGUMENT, "storage_config.minio_access_key is empty."); + return Status(SERVER_INVALID_ARGUMENT, "storage_config.s3_access_key is empty."); } return Status::OK(); } Status -Config::CheckStorageConfigMinioSecretKey(const std::string& value) { +Config::CheckStorageConfigS3SecretKey(const std::string& value) { if (value.empty()) { - return Status(SERVER_INVALID_ARGUMENT, "storage_config.minio_secret_key is empty."); + return Status(SERVER_INVALID_ARGUMENT, "storage_config.s3_secret_key is empty."); } return Status::OK(); } Status -Config::CheckStorageConfigMinioBucket(const std::string& value) { +Config::CheckStorageConfigS3Bucket(const std::string& value) { if (value.empty()) { - return Status(SERVER_INVALID_ARGUMENT, "storage_config.minio_bucket is empty."); + return Status(SERVER_INVALID_ARGUMENT, "storage_config.s3_bucket is empty."); } return Status::OK(); } @@ -972,41 +971,41 @@ Config::GetStorageConfigSecondaryPath(std::string& value) { } Status -Config::GetStorageConfigMinioEnable(bool& value) { - std::string str = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_MINIO_ENABLE, CONFIG_STORAGE_MINIO_ENABLE_DEFAULT); - CONFIG_CHECK(CheckStorageConfigMinioEnable(str)); +Config::GetStorageConfigS3Enable(bool& value) { + std::string str = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_S3_ENABLE, CONFIG_STORAGE_S3_ENABLE_DEFAULT); + CONFIG_CHECK(CheckStorageConfigS3Enable(str)); std::transform(str.begin(), str.end(), str.begin(), ::tolower); value = (str == "true" || str == "on" || str == "yes" || str == "1"); return Status::OK(); } Status -Config::GetStorageConfigMinioAddress(std::string& value) { - value = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_MINIO_ADDRESS, CONFIG_STORAGE_MINIO_ADDRESS_DEFAULT); - return CheckStorageConfigMinioAddress(value); +Config::GetStorageConfigS3Address(std::string& value) { + value = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_S3_ADDRESS, CONFIG_STORAGE_S3_ADDRESS_DEFAULT); + return CheckStorageConfigS3Address(value); } Status -Config::GetStorageConfigMinioPort(std::string& value) { - value = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_MINIO_PORT, CONFIG_STORAGE_MINIO_PORT_DEFAULT); - return CheckStorageConfigMinioPort(value); +Config::GetStorageConfigS3Port(std::string& value) { + value = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_S3_PORT, CONFIG_STORAGE_S3_PORT_DEFAULT); + return CheckStorageConfigS3Port(value); } Status -Config::GetStorageConfigMinioAccessKey(std::string& value) { - value = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_MINIO_ACCESS_KEY, CONFIG_STORAGE_MINIO_ACCESS_KEY_DEFAULT); +Config::GetStorageConfigS3AccessKey(std::string& value) { + value = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_S3_ACCESS_KEY, CONFIG_STORAGE_S3_ACCESS_KEY_DEFAULT); return Status::OK(); } Status -Config::GetStorageConfigMinioSecretKey(std::string& value) { - value = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_MINIO_SECRET_KEY, CONFIG_STORAGE_MINIO_SECRET_KEY_DEFAULT); +Config::GetStorageConfigS3SecretKey(std::string& value) { + value = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_S3_SECRET_KEY, CONFIG_STORAGE_S3_SECRET_KEY_DEFAULT); return Status::OK(); } Status -Config::GetStorageConfigMinioBucket(std::string& value) { - value = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_MINIO_BUCKET, CONFIG_STORAGE_MINIO_BUCKET_DEFAULT); +Config::GetStorageConfigS3Bucket(std::string& value) { + value = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_S3_BUCKET, CONFIG_STORAGE_S3_BUCKET_DEFAULT); return Status::OK(); } @@ -1259,39 +1258,39 @@ Config::SetStorageConfigSecondaryPath(const std::string& value) { } Status -Config::SetStorageConfigMinioEnable(const std::string& value) { - CONFIG_CHECK(CheckStorageConfigMinioEnable(value)); - return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_MINIO_ENABLE, value); +Config::SetStorageConfigS3Enable(const std::string& value) { + CONFIG_CHECK(CheckStorageConfigS3Enable(value)); + return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_S3_ENABLE, value); } Status -Config::SetStorageConfigMinioAddress(const std::string& value) { - CONFIG_CHECK(CheckStorageConfigMinioAddress(value)); - return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_MINIO_ADDRESS, value); +Config::SetStorageConfigS3Address(const std::string& value) { + CONFIG_CHECK(CheckStorageConfigS3Address(value)); + return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_S3_ADDRESS, value); } Status -Config::SetStorageConfigMinioPort(const std::string& value) { - CONFIG_CHECK(CheckStorageConfigMinioPort(value)); - return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_MINIO_PORT, value); +Config::SetStorageConfigS3Port(const std::string& value) { + CONFIG_CHECK(CheckStorageConfigS3Port(value)); + return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_S3_PORT, value); } Status -Config::SetStorageConfigMinioAccessKey(const std::string& value) { - CONFIG_CHECK(CheckStorageConfigMinioAccessKey(value)); - return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_MINIO_ACCESS_KEY, value); +Config::SetStorageConfigS3AccessKey(const std::string& value) { + CONFIG_CHECK(CheckStorageConfigS3AccessKey(value)); + return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_S3_ACCESS_KEY, value); } Status -Config::SetStorageConfigMinioSecretKey(const std::string& value) { - CONFIG_CHECK(CheckStorageConfigMinioSecretKey(value)); - return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_MINIO_SECRET_KEY, value); +Config::SetStorageConfigS3SecretKey(const std::string& value) { + CONFIG_CHECK(CheckStorageConfigS3SecretKey(value)); + return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_S3_SECRET_KEY, value); } Status -Config::SetStorageConfigMinioBucket(const std::string& value) { - CONFIG_CHECK(CheckStorageConfigMinioBucket(value)); - return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_MINIO_BUCKET, value); +Config::SetStorageConfigS3Bucket(const std::string& value) { + CONFIG_CHECK(CheckStorageConfigS3Bucket(value)); + return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_S3_BUCKET, value); } /* metric config */ diff --git a/core/src/server/Config.h b/core/src/server/Config.h index 4401c6d87feb1..172865702b535 100644 --- a/core/src/server/Config.h +++ b/core/src/server/Config.h @@ -71,18 +71,18 @@ static const char* CONFIG_STORAGE_PRIMARY_PATH = "primary_path"; static const char* CONFIG_STORAGE_PRIMARY_PATH_DEFAULT = "/tmp/milvus"; static const char* CONFIG_STORAGE_SECONDARY_PATH = "secondary_path"; static const char* CONFIG_STORAGE_SECONDARY_PATH_DEFAULT = ""; -static const char* CONFIG_STORAGE_MINIO_ENABLE = "minio_enable"; -static const char* CONFIG_STORAGE_MINIO_ENABLE_DEFAULT = "false"; -static const char* CONFIG_STORAGE_MINIO_ADDRESS = "minio_address"; -static const char* CONFIG_STORAGE_MINIO_ADDRESS_DEFAULT = "127.0.0.1"; -static const char* CONFIG_STORAGE_MINIO_PORT = "minio_port"; -static const char* CONFIG_STORAGE_MINIO_PORT_DEFAULT = "9000"; -static const char* CONFIG_STORAGE_MINIO_ACCESS_KEY = "minio_access_key"; -static const char* CONFIG_STORAGE_MINIO_ACCESS_KEY_DEFAULT = "minioadmin"; -static const char* CONFIG_STORAGE_MINIO_SECRET_KEY = "minio_secret_key"; -static const char* CONFIG_STORAGE_MINIO_SECRET_KEY_DEFAULT = "minioadmin"; -static const char* CONFIG_STORAGE_MINIO_BUCKET = "minio_bucket"; -static const char* CONFIG_STORAGE_MINIO_BUCKET_DEFAULT = "milvus-bucket"; +static const char* CONFIG_STORAGE_S3_ENABLE = "s3_enable"; +static const char* CONFIG_STORAGE_S3_ENABLE_DEFAULT = "false"; +static const char* CONFIG_STORAGE_S3_ADDRESS = "s3_address"; +static const char* CONFIG_STORAGE_S3_ADDRESS_DEFAULT = "127.0.0.1"; +static const char* CONFIG_STORAGE_S3_PORT = "s3_port"; +static const char* CONFIG_STORAGE_S3_PORT_DEFAULT = "9000"; +static const char* CONFIG_STORAGE_S3_ACCESS_KEY = "s3_access_key"; +static const char* CONFIG_STORAGE_S3_ACCESS_KEY_DEFAULT = "minioadmin"; +static const char* CONFIG_STORAGE_S3_SECRET_KEY = "s3_secret_key"; +static const char* CONFIG_STORAGE_S3_SECRET_KEY_DEFAULT = "minioadmin"; +static const char* CONFIG_STORAGE_S3_BUCKET = "s3_bucket"; +static const char* CONFIG_STORAGE_S3_BUCKET_DEFAULT = "milvus-bucket"; /* cache config */ static const char* CONFIG_CACHE = "cache_config"; @@ -197,17 +197,17 @@ class Config { Status CheckStorageConfigSecondaryPath(const std::string& value); Status - CheckStorageConfigMinioEnable(const std::string& value); + CheckStorageConfigS3Enable(const std::string& value); Status - CheckStorageConfigMinioAddress(const std::string& value); + CheckStorageConfigS3Address(const std::string& value); Status - CheckStorageConfigMinioPort(const std::string& value); + CheckStorageConfigS3Port(const std::string& value); Status - CheckStorageConfigMinioAccessKey(const std::string& value); + CheckStorageConfigS3AccessKey(const std::string& value); Status - CheckStorageConfigMinioSecretKey(const std::string& value); + CheckStorageConfigS3SecretKey(const std::string& value); Status - CheckStorageConfigMinioBucket(const std::string& value); + CheckStorageConfigS3Bucket(const std::string& value); /* metric config */ Status @@ -287,17 +287,17 @@ class Config { Status GetStorageConfigSecondaryPath(std::string& value); Status - GetStorageConfigMinioEnable(bool& value); + GetStorageConfigS3Enable(bool& value); Status - GetStorageConfigMinioAddress(std::string& value); + GetStorageConfigS3Address(std::string& value); Status - GetStorageConfigMinioPort(std::string& value); + GetStorageConfigS3Port(std::string& value); Status - GetStorageConfigMinioAccessKey(std::string& value); + GetStorageConfigS3AccessKey(std::string& value); Status - GetStorageConfigMinioSecretKey(std::string& value); + GetStorageConfigS3SecretKey(std::string& value); Status - GetStorageConfigMinioBucket(std::string& value); + GetStorageConfigS3Bucket(std::string& value); /* metric config */ Status @@ -371,17 +371,17 @@ class Config { Status SetStorageConfigSecondaryPath(const std::string& value); Status - SetStorageConfigMinioEnable(const std::string& value); + SetStorageConfigS3Enable(const std::string& value); Status - SetStorageConfigMinioAddress(const std::string& value); + SetStorageConfigS3Address(const std::string& value); Status - SetStorageConfigMinioPort(const std::string& value); + SetStorageConfigS3Port(const std::string& value); Status - SetStorageConfigMinioAccessKey(const std::string& value); + SetStorageConfigS3AccessKey(const std::string& value); Status - SetStorageConfigMinioSecretKey(const std::string& value); + SetStorageConfigS3SecretKey(const std::string& value); Status - SetStorageConfigMinioBucket(const std::string& value); + SetStorageConfigS3Bucket(const std::string& value); /* metric config */ Status diff --git a/core/src/storage/s3/S3ClientWrapper.cpp b/core/src/storage/s3/S3ClientWrapper.cpp index e7d9251b9f1ac..ce7e158ab65c9 100644 --- a/core/src/storage/s3/S3ClientWrapper.cpp +++ b/core/src/storage/s3/S3ClientWrapper.cpp @@ -40,28 +40,28 @@ namespace storage { Status S3ClientWrapper::StartService() { server::Config& config = server::Config::GetInstance(); - bool minio_enable = false; - CONFIG_CHECK(config.GetStorageConfigMinioEnable(minio_enable)); - fiu_do_on("S3ClientWrapper.StartService.minio_disable", minio_enable = false); - if (!minio_enable) { - STORAGE_LOG_INFO << "MinIO not enabled!"; + bool s3_enable = false; + CONFIG_CHECK(config.GetStorageConfigS3Enable(s3_enable)); + fiu_do_on("S3ClientWrapper.StartService.s3_disable", s3_enable = false); + if (!s3_enable) { + STORAGE_LOG_INFO << "S3 not enabled!"; return Status::OK(); } - CONFIG_CHECK(config.GetStorageConfigMinioAddress(minio_address_)); - CONFIG_CHECK(config.GetStorageConfigMinioPort(minio_port_)); - CONFIG_CHECK(config.GetStorageConfigMinioAccessKey(minio_access_key_)); - CONFIG_CHECK(config.GetStorageConfigMinioSecretKey(minio_secret_key_)); - CONFIG_CHECK(config.GetStorageConfigMinioBucket(minio_bucket_)); + CONFIG_CHECK(config.GetStorageConfigS3Address(s3_address_)); + CONFIG_CHECK(config.GetStorageConfigS3Port(s3_port_)); + CONFIG_CHECK(config.GetStorageConfigS3AccessKey(s3_access_key_)); + CONFIG_CHECK(config.GetStorageConfigS3SecretKey(s3_secret_key_)); + CONFIG_CHECK(config.GetStorageConfigS3Bucket(s3_bucket_)); Aws::InitAPI(options_); Aws::Client::ClientConfiguration cfg; - cfg.endpointOverride = minio_address_ + ":" + minio_port_; + cfg.endpointOverride = s3_address_ + ":" + s3_port_; cfg.scheme = Aws::Http::Scheme::HTTP; cfg.verifySSL = false; client_ptr_ = - std::make_shared(Aws::Auth::AWSCredentials(minio_access_key_, minio_secret_key_), cfg, + std::make_shared(Aws::Auth::AWSCredentials(s3_access_key_, s3_secret_key_), cfg, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Always, false); bool mock_enable = false; @@ -84,7 +84,7 @@ S3ClientWrapper::StopService() { Status S3ClientWrapper::CreateBucket() { Aws::S3::Model::CreateBucketRequest request; - request.WithBucket(minio_bucket_); + request.WithBucket(s3_bucket_); auto outcome = client_ptr_->CreateBucket(request); @@ -97,14 +97,14 @@ S3ClientWrapper::CreateBucket() { } } - STORAGE_LOG_DEBUG << "CreateBucket '" << minio_bucket_ << "' successfully!"; + STORAGE_LOG_DEBUG << "CreateBucket '" << s3_bucket_ << "' successfully!"; return Status::OK(); } Status S3ClientWrapper::DeleteBucket() { Aws::S3::Model::DeleteBucketRequest request; - request.WithBucket(minio_bucket_); + request.WithBucket(s3_bucket_); auto outcome = client_ptr_->DeleteBucket(request); @@ -115,7 +115,7 @@ S3ClientWrapper::DeleteBucket() { return Status(SERVER_UNEXPECTED_ERROR, err.GetMessage()); } - STORAGE_LOG_DEBUG << "DeleteBucket '" << minio_bucket_ << "' successfully!"; + STORAGE_LOG_DEBUG << "DeleteBucket '" << s3_bucket_ << "' successfully!"; return Status::OK(); } @@ -129,7 +129,7 @@ S3ClientWrapper::PutObjectFile(const std::string& object_name, const std::string } Aws::S3::Model::PutObjectRequest request; - request.WithBucket(minio_bucket_).WithKey(object_name); + request.WithBucket(s3_bucket_).WithKey(object_name); auto input_data = Aws::MakeShared("PutObjectFile", file_path.c_str(), std::ios_base::in | std::ios_base::binary); @@ -151,7 +151,7 @@ S3ClientWrapper::PutObjectFile(const std::string& object_name, const std::string Status S3ClientWrapper::PutObjectStr(const std::string& object_name, const std::string& content) { Aws::S3::Model::PutObjectRequest request; - request.WithBucket(minio_bucket_).WithKey(object_name); + request.WithBucket(s3_bucket_).WithKey(object_name); const std::shared_ptr input_data = Aws::MakeShared(""); input_data->write(content.data(), content.length()); @@ -173,7 +173,7 @@ S3ClientWrapper::PutObjectStr(const std::string& object_name, const std::string& Status S3ClientWrapper::GetObjectFile(const std::string& object_name, const std::string& file_path) { Aws::S3::Model::GetObjectRequest request; - request.WithBucket(minio_bucket_).WithKey(object_name); + request.WithBucket(s3_bucket_).WithKey(object_name); auto outcome = client_ptr_->GetObject(request); @@ -196,7 +196,7 @@ S3ClientWrapper::GetObjectFile(const std::string& object_name, const std::string Status S3ClientWrapper::GetObjectStr(const std::string& object_name, std::string& content) { Aws::S3::Model::GetObjectRequest request; - request.WithBucket(minio_bucket_).WithKey(object_name); + request.WithBucket(s3_bucket_).WithKey(object_name); auto outcome = client_ptr_->GetObject(request); @@ -219,7 +219,7 @@ S3ClientWrapper::GetObjectStr(const std::string& object_name, std::string& conte Status S3ClientWrapper::ListObjects(std::vector& object_list, const std::string& marker) { Aws::S3::Model::ListObjectsRequest request; - request.WithBucket(minio_bucket_); + request.WithBucket(s3_bucket_); if (!marker.empty()) { request.WithMarker(marker); @@ -241,9 +241,9 @@ S3ClientWrapper::ListObjects(std::vector& object_list, const std::s } if (marker.empty()) { - STORAGE_LOG_DEBUG << "ListObjects '" << minio_bucket_ << "' successfully!"; + STORAGE_LOG_DEBUG << "ListObjects '" << s3_bucket_ << "' successfully!"; } else { - STORAGE_LOG_DEBUG << "ListObjects '" << minio_bucket_ << ":" << marker << "' successfully!"; + STORAGE_LOG_DEBUG << "ListObjects '" << s3_bucket_ << ":" << marker << "' successfully!"; } return Status::OK(); } @@ -251,7 +251,7 @@ S3ClientWrapper::ListObjects(std::vector& object_list, const std::s Status S3ClientWrapper::DeleteObject(const std::string& object_name) { Aws::S3::Model::DeleteObjectRequest request; - request.WithBucket(minio_bucket_).WithKey(object_name); + request.WithBucket(s3_bucket_).WithKey(object_name); auto outcome = client_ptr_->DeleteObject(request); diff --git a/core/src/storage/s3/S3ClientWrapper.h b/core/src/storage/s3/S3ClientWrapper.h index 0e1a99e77de59..60d47e570cee9 100644 --- a/core/src/storage/s3/S3ClientWrapper.h +++ b/core/src/storage/s3/S3ClientWrapper.h @@ -63,11 +63,11 @@ class S3ClientWrapper : public IStorage { std::shared_ptr client_ptr_; Aws::SDKOptions options_; - std::string minio_address_; - std::string minio_port_; - std::string minio_access_key_; - std::string minio_secret_key_; - std::string minio_bucket_; + std::string s3_address_; + std::string s3_port_; + std::string s3_access_key_; + std::string s3_secret_key_; + std::string s3_bucket_; }; } // namespace storage diff --git a/core/src/wrapper/VecIndex.cpp b/core/src/wrapper/VecIndex.cpp index cc90c87d190e0..e86e1e30ebaa9 100644 --- a/core/src/wrapper/VecIndex.cpp +++ b/core/src/wrapper/VecIndex.cpp @@ -167,12 +167,12 @@ read_index(const std::string& location) { TimeRecorder recorder("read_index"); knowhere::BinarySet load_data_list; - bool minio_enable = false; + bool s3_enable = false; server::Config& config = server::Config::GetInstance(); - config.GetStorageConfigMinioEnable(minio_enable); + config.GetStorageConfigS3Enable(s3_enable); std::shared_ptr reader_ptr; - if (minio_enable) { + if (s3_enable) { reader_ptr = std::make_shared(location); } else { reader_ptr = std::make_shared(location); @@ -235,12 +235,12 @@ write_index(VecIndexPtr index, const std::string& location) { auto binaryset = index->Serialize(); auto index_type = index->GetType(); - bool minio_enable = false; + bool s3_enable = false; server::Config& config = server::Config::GetInstance(); - config.GetStorageConfigMinioEnable(minio_enable); + config.GetStorageConfigS3Enable(s3_enable); std::shared_ptr writer_ptr; - if (minio_enable) { + if (s3_enable) { writer_ptr = std::make_shared(location); } else { writer_ptr = std::make_shared(location); diff --git a/core/unittest/server/test_config.cpp b/core/unittest/server/test_config.cpp index 7b3aba30aa806..8785f4e33d3e2 100644 --- a/core/unittest/server/test_config.cpp +++ b/core/unittest/server/test_config.cpp @@ -182,35 +182,35 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { ASSERT_TRUE(config.GetStorageConfigSecondaryPath(str_val).ok()); ASSERT_TRUE(str_val == storage_secondary_path); - bool storage_minio_enable = true; - ASSERT_TRUE(config.SetStorageConfigMinioEnable(std::to_string(storage_minio_enable)).ok()); - ASSERT_TRUE(config.GetStorageConfigMinioEnable(bool_val).ok()); - ASSERT_TRUE(bool_val == storage_minio_enable); - - std::string storage_minio_addr = "192.168.1.100"; - ASSERT_TRUE(config.SetStorageConfigMinioAddress(storage_minio_addr).ok()); - ASSERT_TRUE(config.GetStorageConfigMinioAddress(str_val).ok()); - ASSERT_TRUE(str_val == storage_minio_addr); - - std::string storage_minio_port = "12345"; - ASSERT_TRUE(config.SetStorageConfigMinioPort(storage_minio_port).ok()); - ASSERT_TRUE(config.GetStorageConfigMinioPort(str_val).ok()); - ASSERT_TRUE(str_val == storage_minio_port); - - std::string storage_minio_access_key = "minioadmin"; - ASSERT_TRUE(config.SetStorageConfigMinioAccessKey(storage_minio_access_key).ok()); - ASSERT_TRUE(config.GetStorageConfigMinioAccessKey(str_val).ok()); - ASSERT_TRUE(str_val == storage_minio_access_key); - - std::string storage_minio_secret_key = "minioadmin"; - ASSERT_TRUE(config.SetStorageConfigMinioSecretKey(storage_minio_secret_key).ok()); - ASSERT_TRUE(config.GetStorageConfigMinioSecretKey(str_val).ok()); - ASSERT_TRUE(str_val == storage_minio_secret_key); - - std::string storage_minio_bucket = "miniobucket"; - ASSERT_TRUE(config.SetStorageConfigMinioBucket(storage_minio_bucket).ok()); - ASSERT_TRUE(config.GetStorageConfigMinioBucket(str_val).ok()); - ASSERT_TRUE(str_val == storage_minio_bucket); + bool storage_s3_enable = true; + ASSERT_TRUE(config.SetStorageConfigS3Enable(std::to_string(storage_s3_enable)).ok()); + ASSERT_TRUE(config.GetStorageConfigS3Enable(bool_val).ok()); + ASSERT_TRUE(bool_val == storage_s3_enable); + + std::string storage_s3_addr = "192.168.1.100"; + ASSERT_TRUE(config.SetStorageConfigS3Address(storage_s3_addr).ok()); + ASSERT_TRUE(config.GetStorageConfigS3Address(str_val).ok()); + ASSERT_TRUE(str_val == storage_s3_addr); + + std::string storage_s3_port = "12345"; + ASSERT_TRUE(config.SetStorageConfigS3Port(storage_s3_port).ok()); + ASSERT_TRUE(config.GetStorageConfigS3Port(str_val).ok()); + ASSERT_TRUE(str_val == storage_s3_port); + + std::string storage_s3_access_key = "minioadmin"; + ASSERT_TRUE(config.SetStorageConfigS3AccessKey(storage_s3_access_key).ok()); + ASSERT_TRUE(config.GetStorageConfigS3AccessKey(str_val).ok()); + ASSERT_TRUE(str_val == storage_s3_access_key); + + std::string storage_s3_secret_key = "minioadmin"; + ASSERT_TRUE(config.SetStorageConfigS3SecretKey(storage_s3_secret_key).ok()); + ASSERT_TRUE(config.GetStorageConfigS3SecretKey(str_val).ok()); + ASSERT_TRUE(str_val == storage_s3_secret_key); + + std::string storage_s3_bucket = "s3bucket"; + ASSERT_TRUE(config.SetStorageConfigS3Bucket(storage_s3_bucket).ok()); + ASSERT_TRUE(config.GetStorageConfigS3Bucket(str_val).ok()); + ASSERT_TRUE(str_val == storage_s3_bucket); /* metric config */ bool metric_enable_monitor = false; @@ -354,9 +354,9 @@ TEST_F(ConfigTest, SERVER_CONFIG_CLI_TEST) { ASSERT_TRUE(s.ok()); /* storage config */ - std::string storage_minio_enable = "true"; - get_cmd = gen_get_command(ms::CONFIG_STORAGE, ms::CONFIG_STORAGE_MINIO_ENABLE); - set_cmd = gen_set_command(ms::CONFIG_STORAGE, ms::CONFIG_STORAGE_MINIO_ENABLE, storage_minio_enable); + std::string storage_s3_enable = "true"; + get_cmd = gen_get_command(ms::CONFIG_STORAGE, ms::CONFIG_STORAGE_S3_ENABLE); + set_cmd = gen_set_command(ms::CONFIG_STORAGE, ms::CONFIG_STORAGE_S3_ENABLE, storage_s3_enable); s = config.ProcessConfigCli(dummy, set_cmd); ASSERT_FALSE(s.ok()); s = config.ProcessConfigCli(result, get_cmd); @@ -512,18 +512,18 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { // ASSERT_FALSE(config.SetStorageConfigSecondaryPath("").ok()); - ASSERT_FALSE(config.SetStorageConfigMinioEnable("10").ok()); + ASSERT_FALSE(config.SetStorageConfigS3Enable("10").ok()); - ASSERT_FALSE(config.SetStorageConfigMinioAddress("127.0.0").ok()); + ASSERT_FALSE(config.SetStorageConfigS3Address("127.0.0").ok()); - ASSERT_FALSE(config.SetStorageConfigMinioPort("100").ok()); - ASSERT_FALSE(config.SetStorageConfigMinioPort("100000").ok()); + ASSERT_FALSE(config.SetStorageConfigS3Port("100").ok()); + ASSERT_FALSE(config.SetStorageConfigS3Port("100000").ok()); - ASSERT_FALSE(config.SetStorageConfigMinioAccessKey("").ok()); + ASSERT_FALSE(config.SetStorageConfigS3AccessKey("").ok()); - ASSERT_FALSE(config.SetStorageConfigMinioSecretKey("").ok()); + ASSERT_FALSE(config.SetStorageConfigS3SecretKey("").ok()); - ASSERT_FALSE(config.SetStorageConfigMinioBucket("").ok()); + ASSERT_FALSE(config.SetStorageConfigS3Bucket("").ok()); /* metric config */ ASSERT_FALSE(config.SetMetricConfigEnableMonitor("Y").ok()); diff --git a/core/unittest/storage/test_s3_client.cpp b/core/unittest/storage/test_s3_client.cpp index c4899eaddd65e..6db6f7d6bc1c6 100644 --- a/core/unittest/storage/test_s3_client.cpp +++ b/core/unittest/storage/test_s3_client.cpp @@ -144,9 +144,9 @@ TEST_F(StorageTest, S3_FAIL_TEST) { auto& storage_inst = milvus::storage::S3ClientWrapper::GetInstance(); - fiu_enable("S3ClientWrapper.StartService.minio_disable", 1, NULL, 0); + fiu_enable("S3ClientWrapper.StartService.s3_disable", 1, NULL, 0); ASSERT_TRUE(storage_inst.StartService().ok()); - fiu_disable("S3ClientWrapper.StartService.minio_disable"); + fiu_disable("S3ClientWrapper.StartService.s3_disable"); fiu_enable("S3ClientWrapper.StartService.mock_enable", 1, NULL, 0); ASSERT_TRUE(storage_inst.StartService().ok()); diff --git a/core/unittest/storage/utils.cpp b/core/unittest/storage/utils.cpp index 4d80416bb7107..d4bda45b7f22a 100644 --- a/core/unittest/storage/utils.cpp +++ b/core/unittest/storage/utils.cpp @@ -28,12 +28,12 @@ static const char* CONFIG_STR = "storage_config:\n" " primary_path: /tmp/milvus\n" " secondary_path:\n" - " minio_enable: true\n" - " minio_address: 127.0.0.1\n" - " minio_port: 9000\n" - " minio_access_key: minioadmin\n" - " minio_secret_key: minioadmin\n" - " minio_bucket: milvus-bucket\n" + " s3_enable: true\n" + " s3_address: 127.0.0.1\n" + " s3_port: 9000\n" + " s3_access_key: minioadmin\n" + " s3_secret_key: minioadmin\n" + " s3_bucket: milvus-bucket\n" "\n"; void