[#5220] improvment(hadoop-catalog): Optimize the name properties keys for Hadoop catalog.

bundles/aliyun-bundle/build.gradle.kts

@@ -35,6 +35,7 @@ dependencies {
   // org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem.initialize(AliyunOSSFileSystem.java:323)
   // org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3611)
   implementation(libs.commons.lang)
+  implementation(project(":catalogs:catalog-common"))
 }
 
 tasks.withType(ShadowJar::class.java) {

bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/fs/OSSFileSystemProvider.java

@@ -18,23 +18,41 @@
  */
 package org.apache.gravitino.oss.fs;
 
+import com.google.common.collect.ImmutableMap;
 import java.io.IOException;
 import java.util.Map;
 import org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider;
+import org.apache.gravitino.storage.OSSProperties;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem;
+import org.apache.hadoop.fs.aliyun.oss.Constants;
 
 public class OSSFileSystemProvider implements FileSystemProvider {
+
+  private static final String OSS_FILESYSTEM_IMPL = "fs.oss.impl";
+
+  public static final Map<String, String> GRAVITINO_KEY_TO_OSS_HADOOP_KEY =
+      ImmutableMap.of(
+          OSSProperties.GRAVITINO_OSS_ENDPOINT, Constants.ENDPOINT_KEY,
+          OSSProperties.GRAVITINO_OSS_ACCESS_KEY_ID, Constants.ACCESS_KEY_ID,
+          OSSProperties.GRAVITINO_OSS_ACCESS_KEY_SECRET, Constants.ACCESS_KEY_SECRET);
+
   @Override
   public FileSystem getFileSystem(Path path, Map<String, String> config) throws IOException {
     Configuration configuration = new Configuration();
     config.forEach(
         (k, v) -> {
-          configuration.set(k.replace("gravitino.bypass.", ""), v);
+          if (k.startsWith(GRAVITINO_BYPASS)) {
+            configuration.set(k.replace(GRAVITINO_BYPASS, ""), v);
+          } else configuration.set(GRAVITINO_KEY_TO_OSS_HADOOP_KEY.getOrDefault(k, k), v);
         });
 
+    // OSS do not use service loader to load the file system, so we need to set the impl class
+    if (!config.containsKey(OSS_FILESYSTEM_IMPL)) {
+      configuration.set(OSS_FILESYSTEM_IMPL, AliyunOSSFileSystem.class.getCanonicalName());
+    }
     return AliyunOSSFileSystem.newInstance(path.toUri(), configuration);
   }
 

bundles/aws-bundle/build.gradle.kts

@@ -28,6 +28,7 @@ dependencies {
   compileOnly(project(":catalogs:catalog-hadoop"))
   compileOnly(libs.hadoop3.common)
   implementation(libs.hadoop3.aws)
+  implementation(project(":catalogs:catalog-common"))
 }
 
 tasks.withType(ShadowJar::class.java) {

bundles/aws-bundle/src/main/java/org/apache/gravitino/s3/fs/S3FileSystemProvider.java

@@ -19,23 +19,37 @@
 
 package org.apache.gravitino.s3.fs;
 
+import com.google.common.collect.ImmutableMap;
 import java.io.IOException;
 import java.util.Map;
 import org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider;
+import org.apache.gravitino.storage.S3Properties;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.Constants;
 import org.apache.hadoop.fs.s3a.S3AFileSystem;
 
 public class S3FileSystemProvider implements FileSystemProvider {
+
+  public static final Map<String, String> GRAVITINO_KEY_TO_S3_HADOOP_KEY =
+      ImmutableMap.of(
+          S3Properties.GRAVITINO_S3_ENDPOINT, Constants.ENDPOINT,
+          S3Properties.GRAVITINO_S3_ACCESS_KEY_ID, Constants.ACCESS_KEY,
+          S3Properties.GRAVITINO_S3_SECRET_ACCESS_KEY, Constants.SECRET_KEY);
+
   @Override
   public FileSystem getFileSystem(Path path, Map<String, String> config) throws IOException {
     Configuration configuration = new Configuration();
     config.forEach(
         (k, v) -> {
-          configuration.set(k.replace("gravitino.bypass.", ""), v);
+          if (k.startsWith(GRAVITINO_BYPASS)) {
+            configuration.set(k.replace(GRAVITINO_BYPASS, ""), v);
+          } else configuration.set(GRAVITINO_KEY_TO_S3_HADOOP_KEY.getOrDefault(k, k), v);
         });
 
+    configuration.set(
+        Constants.AWS_CREDENTIALS_PROVIDER, Constants.ASSUMED_ROLE_CREDENTIALS_DEFAULT);
     return S3AFileSystem.newInstance(path.toUri(), configuration);
   }
 

bundles/gcp-bundle/build.gradle.kts

@@ -34,6 +34,7 @@ dependencies {
 
   implementation(libs.commons.lang3)
   implementation(libs.hadoop3.gcs)
+  implementation(project(":catalogs:catalog-common"))
   implementation(libs.google.auth.http)
   implementation(libs.google.auth.credentials)
 }

bundles/gcp-bundle/src/main/java/org/apache/gravitino/gcs/fs/GCSFileSystemProvider.java

@@ -19,9 +19,11 @@
 package org.apache.gravitino.gcs.fs;
 
 import com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem;
+import com.google.common.collect.ImmutableMap;
 import java.io.IOException;
 import java.util.Map;
 import org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider;
+import org.apache.gravitino.storage.GCSProperties;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -31,12 +33,18 @@
 public class GCSFileSystemProvider implements FileSystemProvider {
   private static final Logger LOGGER = LoggerFactory.getLogger(GCSFileSystemProvider.class);
 
+  public static final Map<String, String> GRAVITINO_KEY_TO_GCS_HADOOP_KEY =
+      ImmutableMap.of(
+          GCSProperties.GCS_SERVICE_ACCOUNT_JSON_PATH, "fs.gs.auth.service.account.json.keyfile");
+
   @Override
   public FileSystem getFileSystem(Path path, Map<String, String> config) throws IOException {
     Configuration configuration = new Configuration();
     config.forEach(
         (k, v) -> {
-          configuration.set(k.replace("gravitino.bypass.", ""), v);
+          if (k.startsWith(GRAVITINO_BYPASS)) {
+            configuration.set(k.replace(GRAVITINO_BYPASS, ""), v);
+          } else configuration.set(GRAVITINO_KEY_TO_GCS_HADOOP_KEY.getOrDefault(k, k), v);
         });
 
     LOGGER.info("Creating GCS file system with config: {}", config);

catalogs/catalog-common/src/main/java/org/apache/gravitino/storage/GCSProperties.java

@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.gravitino.storage;
+
+public class GCSProperties {
+
+  // The path of service account JSON file of Google Cloud Storage.
+  public static final String GCS_SERVICE_ACCOUNT_JSON_PATH = "gcs-service-account-file";
+
+  private GCSProperties() {}
+}

catalogs/catalog-common/src/main/java/org/apache/gravitino/storage/OSSProperties.java

@@ -26,7 +26,7 @@ public class OSSProperties {
   // The static access key ID used to access OSS data.
   public static final String GRAVITINO_OSS_ACCESS_KEY_ID = "oss-access-key-id";
   // The static access key secret used to access OSS data.
-  public static final String GRAVITINO_OSS_ACCESS_KEY_SECRET = "oss-access-key-secret";
+  public static final String GRAVITINO_OSS_ACCESS_KEY_SECRET = "oss-secret-access-key";
 
   private OSSProperties() {}
 }

catalogs/catalog-common/src/main/java/org/apache/gravitino/storage/S3Properties.java

@@ -31,5 +31,8 @@ public class S3Properties {
   // The region of the S3 service.
   public static final String GRAVITINO_S3_REGION = "s3-region";
 
+  // The S3 credentials provider class name.
+  public static final String GRAVITINO_S3_CREDS_PROVIDER = "s3-creds-provider";
+
   private S3Properties() {}
 }

catalogs/catalog-hadoop/build.gradle.kts

@@ -36,6 +36,10 @@ dependencies {
     exclude(group = "*")
   }
 
+  implementation(project(":catalogs:catalog-common")) {
+    exclude(group = "*")
+  }
+
   compileOnly(libs.guava)
 
   implementation(libs.hadoop3.common) {

catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemProvider.java

@@ -32,6 +32,8 @@
  */
 public interface FileSystemProvider {
 
+  String GRAVITINO_BYPASS = "gravitino.bypass.";
+
   /**
    * Get the FileSystem instance according to the configuration map and file path.
    *

catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCSCatalogIT.java

@@ -26,6 +26,7 @@
 import java.util.Map;
 import org.apache.gravitino.Catalog;
 import org.apache.gravitino.integration.test.util.GravitinoITUtils;
+import org.apache.gravitino.storage.GCSProperties;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -95,10 +96,8 @@ protected String defaultBaseLocation() {
 
   protected void createCatalog() {
     Map<String, String> map = Maps.newHashMap();
-    map.put("gravitino.bypass.fs.gs.auth.service.account.enable", "true");
-    map.put("gravitino.bypass.fs.gs.auth.service.account.json.keyfile", SERVICE_ACCOUNT_FILE);
+    map.put(GCSProperties.GCS_SERVICE_ACCOUNT_JSON_PATH, SERVICE_ACCOUNT_FILE);
     map.put(FILESYSTEM_PROVIDERS, "gcs");
-
     metalake.createCatalog(catalogName, Catalog.Type.FILESET, provider, "comment", map);
 
     catalog = metalake.loadCatalog(catalogName);

catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopOSSCatalogIT.java

@@ -27,6 +27,7 @@
 import java.util.Map;
 import org.apache.gravitino.Catalog;
 import org.apache.gravitino.integration.test.util.GravitinoITUtils;
+import org.apache.gravitino.storage.OSSProperties;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -38,7 +39,8 @@
 
 @Disabled(
     "Disabled due to we don't have a real OSS account to test. If you have a GCP account,"
-        + "please change the configuration(BUCKET_NAME, OSS_ACCESS_KEY, OSS_SECRET_KEY, OSS_ENDPOINT) and enable this test.")
+        + "please change the configuration(BUCKET_NAME, OSS_ACCESS_KEY, OSS_SECRET_KEY, "
+        + "OSS_ENDPOINT) and enable this test.")
 public class HadoopOSSCatalogIT extends HadoopCatalogIT {
   private static final Logger LOG = LoggerFactory.getLogger(HadoopOSSCatalogIT.class);
   public static final String BUCKET_NAME = "YOUR_BUCKET";
@@ -81,8 +83,8 @@ public void setup() throws IOException {
   public void stop() throws IOException {
     Catalog catalog = metalake.loadCatalog(catalogName);
     catalog.asSchemas().dropSchema(schemaName, true);
-    metalake.dropCatalog(catalogName);
-    client.dropMetalake(metalakeName);
+    metalake.dropCatalog(catalogName, true);
+    client.dropMetalake(metalakeName, true);
 
     try {
       closer.close();
@@ -114,10 +116,9 @@ protected String defaultBaseLocation() {
 
   protected void createCatalog() {
     Map<String, String> map = Maps.newHashMap();
-    map.put("gravitino.bypass.fs.oss.accessKeyId", OSS_ACCESS_KEY);
-    map.put("gravitino.bypass.fs.oss.accessKeySecret", OSS_SECRET_KEY);
-    map.put("gravitino.bypass.fs.oss.endpoint", OSS_ENDPOINT);
-    map.put("gravitino.bypass.fs.oss.impl", "org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem");
+    map.put(OSSProperties.GRAVITINO_OSS_ACCESS_KEY_ID, OSS_ACCESS_KEY);
+    map.put(OSSProperties.GRAVITINO_OSS_ACCESS_KEY_SECRET, OSS_SECRET_KEY);
+    map.put(OSSProperties.GRAVITINO_OSS_ENDPOINT, OSS_ENDPOINT);
     map.put(FILESYSTEM_PROVIDERS, "oss");
 
     metalake.createCatalog(catalogName, Catalog.Type.FILESET, provider, "comment", map);

catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopS3CatalogIT.java

@@ -30,6 +30,7 @@
 import org.apache.gravitino.Catalog;
 import org.apache.gravitino.integration.test.container.GravitinoLocalStackContainer;
 import org.apache.gravitino.integration.test.util.GravitinoITUtils;
+import org.apache.gravitino.storage.S3Properties;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -170,12 +171,9 @@ protected String defaultBaseLocation() {
 
   protected void createCatalog() {
     Map<String, String> map = Maps.newHashMap();
-    map.put("gravitino.bypass.fs.s3a.access.key", accessKey);
-    map.put("gravitino.bypass.fs.s3a.secret.key", secretKey);
-    map.put("gravitino.bypass.fs.s3a.endpoint", s3Endpoint);
-    map.put(
-        "gravitino.bypass.fs.s3a.aws.credentials.provider",
-        "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider");
+    map.put(S3Properties.GRAVITINO_S3_ENDPOINT, s3Endpoint);
+    map.put(S3Properties.GRAVITINO_S3_ACCESS_KEY_ID, accessKey);
+    map.put(S3Properties.GRAVITINO_S3_SECRET_ACCESS_KEY, secretKey);
     map.put(FILESYSTEM_PROVIDERS, "s3");
 
     metalake.createCatalog(catalogName, Catalog.Type.FILESET, provider, "comment", map);

clients/client-python/gravitino/filesystem/gvfs_config.py

@@ -32,12 +32,12 @@ class GVFSConfig:
     OAUTH2_PATH = "oauth2_path"
     OAUTH2_SCOPE = "oauth2_scope"
 
-    GVFS_FILESYSTEM_GCS_SERVICE_KEY_FILE = "gcs_service_account_key_path"
+    GVFS_FILESYSTEM_GCS_SERVICE_KEY_FILE = "gcs_service_account_file"
 
-    GVFS_FILESYSTEM_S3_ACCESS_KEY = "s3_access_key"
-    GVFS_FILESYSTEM_S3_SECRET_KEY = "s3_secret_key"
+    GVFS_FILESYSTEM_S3_ACCESS_KEY = "s3_access_key_id"
+    GVFS_FILESYSTEM_S3_SECRET_KEY = "s3_secret_access_key"
     GVFS_FILESYSTEM_S3_ENDPOINT = "s3_endpoint"
 
-    GVFS_FILESYSTEM_OSS_ACCESS_KEY = "oss_access_key"
-    GVFS_FILESYSTEM_OSS_SECRET_KEY = "oss_secret_key"
+    GVFS_FILESYSTEM_OSS_ACCESS_KEY = "oss_access_key_id"
+    GVFS_FILESYSTEM_OSS_SECRET_KEY = "oss_secret_access_key"
     GVFS_FILESYSTEM_OSS_ENDPOINT = "oss_endpoint"

clients/client-python/tests/integration/test_gvfs_with_gcs.py

@@ -109,8 +109,7 @@ def _init_test_entities(cls):
             comment="",
             properties={
                 "filesystem-providers": "gcs",
-                "gravitino.bypass.fs.gs.auth.service.account.enable": "true",
-                "gravitino.bypass.fs.gs.auth.service.account.json.keyfile": cls.key_file,
+                "gcs-service-account-file": cls.key_file,
             },
         )
         catalog.as_schemas().create_schema(

clients/client-python/tests/integration/test_gvfs_with_oss.py

@@ -115,10 +115,9 @@ def _init_test_entities(cls):
             comment="",
             properties={
                 "filesystem-providers": "oss",
-                "gravitino.bypass.fs.oss.accessKeyId": cls.oss_access_key,
-                "gravitino.bypass.fs.oss.accessKeySecret": cls.oss_secret_key,
-                "gravitino.bypass.fs.oss.endpoint": cls.oss_endpoint,
-                "gravitino.bypass.fs.oss.impl": "org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem",
+                "oss-access-key-id": cls.oss_access_key,
+                "oss-secret-access-key": cls.oss_secret_key,
+                "oss-endpoint": cls.oss_endpoint,
             },
         )
         catalog.as_schemas().create_schema(

clients/client-python/tests/integration/test_gvfs_with_s3.py

@@ -113,9 +113,9 @@ def _init_test_entities(cls):
             comment="",
             properties={
                 "filesystem-providers": "s3",
-                "gravitino.bypass.fs.s3a.access.key": cls.s3_access_key,
-                "gravitino.bypass.fs.s3a.secret.key": cls.s3_secret_key,
-                "gravitino.bypass.fs.s3a.endpoint": cls.s3_endpoint,
+                "s3-access-key-id": cls.s3_access_key,
+                "s3-secret-access-key": cls.s3_secret_key,
+                "s3-endpoint": cls.s3_endpoint,
             },
         )
         catalog.as_schemas().create_schema(

clients/filesystem-hadoop3/build.gradle.kts

@@ -29,6 +29,9 @@ dependencies {
   implementation(project(":catalogs:catalog-hadoop")) {
     exclude(group = "*")
   }
+  implementation(project(":catalogs:catalog-common")) {
+    exclude(group = "*")
+  }
 
   implementation(libs.caffeine)
 

clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java

@@ -20,6 +20,7 @@
 
 import static org.apache.gravitino.filesystem.hadoop.GravitinoVirtualFileSystemConfiguration.FS_FILESYSTEM_PROVIDERS;
 import static org.apache.gravitino.filesystem.hadoop.GravitinoVirtualFileSystemConfiguration.GVFS_CONFIG_PREFIX;
+import static org.apache.gravitino.filesystem.hadoop.GravitinoVirtualFileSystemConfiguration.GVFS_KEY_TO_HADOOP_KEY;
 
 import com.github.benmanes.caffeine.cache.Cache;
 import com.github.benmanes.caffeine.cache.Caffeine;
@@ -410,6 +411,8 @@ private Map<String, String> getConfigMap(Configuration configuration) {
           String key = entry.getKey();
           if (key.startsWith(GRAVITINO_BYPASS_PREFIX)) {
             maps.put(key.substring(GRAVITINO_BYPASS_PREFIX.length()), entry.getValue());
+          } else if (GVFS_KEY_TO_HADOOP_KEY.containsKey(key)) {
+            maps.put(GVFS_KEY_TO_HADOOP_KEY.get(key), entry.getValue());
           } else if (!key.startsWith(GVFS_CONFIG_PREFIX)) {
             maps.put(key, entry.getValue());
           }