[Feature] Support aliyun cloud credential (StarRocks#24805)

Signed-off-by: miomiocat <284487410@qq.com>

be/src/fs/credential/cloud_configuration.h

@@ -41,6 +41,17 @@ class AWSCloudCredential final : public CloudCredential {
     }
 };
 
+class AliyunCloudCredential final : public CloudCredential {
+public:
+    std::string access_key;
+    std::string secret_key;
+    std::string endpoint;
+
+    bool operator==(const AliyunCloudCredential& rhs) const {
+        return access_key == rhs.access_key && secret_key == rhs.secret_key && endpoint == rhs.endpoint;
+    }
+};
+
 class CloudConfiguration {
 public:
     virtual ~CloudConfiguration() = default;
@@ -56,4 +67,12 @@ class AWSCloudConfiguration final : public CloudConfiguration {
     bool enable_path_style_access = false;
     bool enable_ssl = true;
 };
+
+class AliyunCloudConfiguration final : public CloudConfiguration {
+public:
+    bool operator==(const AliyunCloudConfiguration& rhs) const {
+        return aliyun_cloud_credential == rhs.aliyun_cloud_credential;
+    }
+    AliyunCloudCredential aliyun_cloud_credential;
+};
 } // namespace starrocks

be/src/fs/credential/cloud_configuration_factory.h

@@ -49,6 +49,10 @@ static const std::string AWS_S3_ENABLE_PATH_STYLE_ACCESS = "aws.s3.enable_path_s
      */
 static const std::string AWS_S3_ENABLE_SSL = "aws.s3.enable_ssl";
 
+static const std::string ALIYUN_OSS_ACCESS_KEY = "aliyun.oss.access_key_id";
+static const std::string ALIYUN_OSS_SECRET_KEY = "aliyun.oss.access_key_secret";
+static const std::string ALIYUN_OSS_ENDPOINT = "aliyun.oss.endpoint";
+
 class CloudConfigurationFactory {
 public:
     static const AWSCloudConfiguration create_aws(const TCloudConfiguration& t_cloud_configuration) {
@@ -80,6 +84,24 @@ class CloudConfigurationFactory {
         return aws_cloud_configuration;
     }
 
+    // This is a reserved interface for aliyun EMR starrocks, and cannot be deleted
+    static const AliyunCloudConfiguration create_aliyun(const TCloudConfiguration& t_cloud_configuration) {
+        DCHECK(t_cloud_configuration.__isset.cloud_type);
+        DCHECK(t_cloud_configuration.cloud_type == TCloudType::ALIYUN);
+        std::unordered_map<std::string, std::string> properties;
+        _insert_properties(properties, t_cloud_configuration);
+
+        AliyunCloudConfiguration aliyun_cloud_configuration{};
+        AliyunCloudCredential aliyun_cloud_credential{};
+
+        aliyun_cloud_credential.access_key = get_or_default(properties, ALIYUN_OSS_ACCESS_KEY, std::string());
+        aliyun_cloud_credential.secret_key = get_or_default(properties, ALIYUN_OSS_SECRET_KEY, std::string());
+        aliyun_cloud_credential.endpoint = get_or_default(properties, ALIYUN_OSS_ENDPOINT, std::string());
+
+        aliyun_cloud_configuration.aliyun_cloud_credential = aliyun_cloud_credential;
+        return aliyun_cloud_configuration;
+    }
+
 private:
     static void _insert_properties(std::unordered_map<std::string, std::string>& properties,
                                    const TCloudConfiguration& t_cloud_configuration) {

be/src/fs/fs_util.h

@@ -163,10 +163,6 @@ inline bool is_gcs_uri(std::string_view uri) {
     return starts_with(uri, "gs://");
 }
 
-inline bool is_jfs_uri(std::string_view uri) {
-    return starts_with(uri, "jfs://");
-}
-
 inline bool is_hdfs_uri(std::string_view uri) {
     return starts_with(uri, "hdfs://");
 }

fe/fe-core/src/main/java/com/starrocks/credential/CloudConfigurationConstants.java

@@ -93,4 +93,9 @@ public class CloudConfigurationConstants {
     public static final String HDFS_KERBEROS_PRINCIPAL = "kerberos_principal";
     public static final String HDFS_KERBEROS_KEYTAB = "kerberos_keytab";
     public static final String HDFS_KERBEROS_KEYTAB_CONTENT = "kerberos_keytab_content";
+
+    // Credential for Aliyun OSS
+    public static final String ALIYUN_OSS_ACCESS_KEY = "aliyun.oss.access_key";
+    public static final String ALIYUN_OSS_SECRET_KEY = "aliyun.oss.secret_key";
+    public static final String ALIYUN_OSS_ENDPOINT = "aliyun.oss.endpoint";
 }

fe/fe-core/src/main/java/com/starrocks/credential/CloudCredentialUtil.java

@@ -44,6 +44,10 @@ public static void maskCloudCredential(Map<String, String> properties) {
 
         // Mask for gcs's credential
         doMask(properties, CloudConfigurationConstants.GCP_GCS_SERVICE_ACCOUNT_PRIVATE_KEY);
+
+        // Mask for oss's credential
+        doMask(properties, CloudConfigurationConstants.ALIYUN_OSS_ACCESS_KEY);
+        doMask(properties, CloudConfigurationConstants.ALIYUN_OSS_SECRET_KEY);
     }
 
     private static void doMask(Map<String, String> properties, String configKey) {

fe/fe-core/src/main/java/com/starrocks/credential/CloudType.java

@@ -20,5 +20,6 @@ public enum CloudType {
     AWS,
     AZURE,
     GCP,
-    HDFS
+    HDFS,
+    ALIYUN
 }

fe/fe-core/src/main/java/com/starrocks/credential/aliyun/AliyunCloudConfiguration.java

@@ -0,0 +1,70 @@
+// Copyright 2021-present StarRocks, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.starrocks.credential.aliyun;
+
+import com.google.common.base.Preconditions;
+import com.staros.proto.FileStoreInfo;
+import com.starrocks.credential.CloudConfiguration;
+import com.starrocks.credential.CloudConfigurationConstants;
+import com.starrocks.credential.CloudType;
+import com.starrocks.thrift.TCloudConfiguration;
+import com.starrocks.thrift.TCloudProperty;
+import com.starrocks.thrift.TCloudType;
+import org.apache.hadoop.conf.Configuration;
+
+import java.util.LinkedList;
+import java.util.List;
+
+public class AliyunCloudConfiguration implements CloudConfiguration {
+
+    private final AliyunCloudCredential aliyunCloudCredential;
+
+    public AliyunCloudConfiguration(AliyunCloudCredential aliyunCloudCredential) {
+        Preconditions.checkNotNull(aliyunCloudCredential);
+        this.aliyunCloudCredential = aliyunCloudCredential;
+    }
+
+    // reuse aws client logic of BE
+    @Override
+    public void toThrift(TCloudConfiguration tCloudConfiguration) {
+        tCloudConfiguration.setCloud_type(TCloudType.AWS);
+
+        List<TCloudProperty> properties = new LinkedList<>();
+        properties.add(new TCloudProperty(CloudConfigurationConstants.AWS_S3_ENABLE_SSL, String.valueOf(true)));
+        aliyunCloudCredential.toThrift(properties);
+        tCloudConfiguration.setCloud_properties(properties);
+    }
+
+    @Override
+    public void applyToConfiguration(Configuration configuration) {
+        aliyunCloudCredential.applyToConfiguration(configuration);
+    }
+
+    @Override
+    public CloudType getCloudType() {
+        return CloudType.ALIYUN;
+    }
+
+    @Override
+    public FileStoreInfo toFileStoreInfo() {
+        // TODO: Support oss credential
+        return aliyunCloudCredential.toFileStoreInfo();
+    }
+
+    @Override
+    public String getCredentialString() {
+        return aliyunCloudCredential.getCredentialString();
+    }
+}

fe/fe-core/src/main/java/com/starrocks/credential/aliyun/AliyunCloudConfigurationFactory.java

@@ -14,11 +14,16 @@
 
 package com.starrocks.credential.aliyun;
 
+import com.google.common.base.Preconditions;
 import com.starrocks.credential.CloudConfiguration;
 import com.starrocks.credential.CloudConfigurationFactory;
 
 import java.util.Map;
 
+import static com.starrocks.credential.CloudConfigurationConstants.ALIYUN_OSS_ACCESS_KEY;
+import static com.starrocks.credential.CloudConfigurationConstants.ALIYUN_OSS_ENDPOINT;
+import static com.starrocks.credential.CloudConfigurationConstants.ALIYUN_OSS_SECRET_KEY;
+
 public class AliyunCloudConfigurationFactory extends CloudConfigurationFactory {
     private final Map<String, String> properties;
 
@@ -28,7 +33,16 @@ public AliyunCloudConfigurationFactory(Map<String, String> properties) {
 
     @Override
     protected CloudConfiguration buildForStorage() {
-        // TODO
-        return null;
+        Preconditions.checkNotNull(properties);
+
+        AliyunCloudCredential aliyunCloudCredential = new AliyunCloudCredential(
+                properties.getOrDefault(ALIYUN_OSS_ACCESS_KEY, ""),
+                properties.getOrDefault(ALIYUN_OSS_SECRET_KEY, ""),
+                properties.getOrDefault(ALIYUN_OSS_ENDPOINT, "")
+        );
+        if (!aliyunCloudCredential.validate()) {
+            return null;
+        }
+        return new AliyunCloudConfiguration(aliyunCloudCredential);
     }
 }

fe/fe-core/src/main/java/com/starrocks/credential/aliyun/AliyunCloudCredential.java

@@ -0,0 +1,76 @@
+// Copyright 2021-present StarRocks, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.starrocks.credential.aliyun;
+
+import com.google.common.base.Preconditions;
+import com.staros.proto.FileStoreInfo;
+import com.starrocks.credential.CloudConfigurationConstants;
+import com.starrocks.credential.CloudCredential;
+import com.starrocks.thrift.TCloudProperty;
+import org.apache.hadoop.conf.Configuration;
+
+import java.util.List;
+
+public class AliyunCloudCredential implements CloudCredential {
+
+    private final String accessKey;
+    private final String secretKey;
+    private final String endpoint;
+
+    public AliyunCloudCredential(String accessKey, String secretKey, String endpoint) {
+        Preconditions.checkNotNull(accessKey);
+        Preconditions.checkNotNull(secretKey);
+        Preconditions.checkNotNull(endpoint);
+        this.accessKey = accessKey;
+        this.secretKey = secretKey;
+        this.endpoint = endpoint;
+    }
+    @Override
+    public void applyToConfiguration(Configuration configuration) {
+        configuration.set("fs.oss.impl", "com.aliyun.jindodata.oss.JindoOssFileSystem");
+        configuration.set("fs.AbstractFileSystem.oss.impl", "com.aliyun.jindodata.oss.OSS");
+        configuration.set("fs.oss.accessKeyId", accessKey);
+        configuration.set("fs.oss.accessKeySecret", secretKey);
+        configuration.set("fs.oss.endpoint", endpoint);
+    }
+
+    @Override
+    public boolean validate() {
+        return !this.accessKey.isEmpty() && !this.secretKey.isEmpty() && !this.endpoint.isEmpty();
+    }
+
+    // reuse aws client logic of BE
+    @Override
+    public void toThrift(List<TCloudProperty> properties) {
+        properties.add(new TCloudProperty(CloudConfigurationConstants.AWS_S3_ACCESS_KEY, accessKey));
+        properties.add(new TCloudProperty(CloudConfigurationConstants.AWS_S3_SECRET_KEY, secretKey));
+        properties.add(new TCloudProperty(CloudConfigurationConstants.AWS_S3_ENDPOINT, endpoint));
+    }
+
+    @Override
+    public String getCredentialString() {
+        return "AliyunCloudCredential{" +
+                "accessKey=" + accessKey +
+                ", secretKey='" + secretKey + '\'' +
+                ", endpoint='" + endpoint + '\'' +
+                '}';
+    }
+
+    @Override
+    public FileStoreInfo toFileStoreInfo() {
+        // TODO: Support oss credential
+        return null;
+    }
+}

gensrc/thrift/CloudConfiguration.thrift

@@ -19,7 +19,8 @@ enum TCloudType {
     DEFAULT,
     AWS,
     AZURE,
-    GCP
+    GCP,
+    ALIYUN
 }
 
 struct TCloudProperty {