Spark: Strip trailing slash from metadatalocation (apache#6121)

Co-authored-by: Prashant Singh <psinghvk@amazon.com>
yabola · Nov 5, 2022 · 6887df3 · 6887df3
1 parent bd225d5
commit 6887df3
Show file tree

Hide file tree

Showing 4 changed files with 13 additions and 9 deletions.
diff --git a/core/src/main/java/org/apache/iceberg/BaseMetastoreTableOperations.java b/core/src/main/java/org/apache/iceberg/BaseMetastoreTableOperations.java
@@ -43,6 +43,7 @@
 import org.apache.iceberg.relocated.com.google.common.base.Objects;
 import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
 import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.apache.iceberg.util.LocationUtil;
 import org.apache.iceberg.util.PropertyUtil;
 import org.apache.iceberg.util.Tasks;
 import org.slf4j.Logger;
@@ -220,7 +221,7 @@ private String metadataFileLocation(TableMetadata metadata, String filename) {
     String metadataLocation = metadata.properties().get(TableProperties.WRITE_METADATA_LOCATION);
 
     if (metadataLocation != null) {
-      return String.format("%s/%s", metadataLocation, filename);
+      return String.format("%s/%s", LocationUtil.stripTrailingSlash(metadataLocation), filename);
     } else {
       return String.format("%s/%s/%s", metadata.location(), METADATA_FOLDER_NAME, filename);
     }

diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTTableOperations.java b/core/src/main/java/org/apache/iceberg/rest/RESTTableOperations.java
@@ -37,6 +37,7 @@
 import org.apache.iceberg.rest.requests.UpdateTableRequest;
 import org.apache.iceberg.rest.responses.ErrorResponse;
 import org.apache.iceberg.rest.responses.LoadTableResponse;
+import org.apache.iceberg.util.LocationUtil;
 
 class RESTTableOperations implements TableOperations {
   private static final String METADATA_FOLDER_NAME = "metadata";
@@ -169,7 +170,7 @@ private static String metadataFileLocation(TableMetadata metadata, String filena
     String metadataLocation = metadata.properties().get(TableProperties.WRITE_METADATA_LOCATION);
 
     if (metadataLocation != null) {
-      return String.format("%s/%s", metadataLocation, filename);
+      return String.format("%s/%s", LocationUtil.stripTrailingSlash(metadataLocation), filename);
     } else {
       return String.format("%s/%s/%s", metadata.location(), METADATA_FOLDER_NAME, filename);
     }

diff --git a/....3/spark/src/main/java/org/apache/iceberg/spark/actions/BaseTableCreationSparkAction.java b/....3/spark/src/main/java/org/apache/iceberg/spark/actions/BaseTableCreationSparkAction.java
@@ -37,6 +37,7 @@
 import org.apache.iceberg.spark.SparkCatalog;
 import org.apache.iceberg.spark.SparkSessionCatalog;
 import org.apache.iceberg.spark.source.StagedSparkTable;
+import org.apache.iceberg.util.LocationUtil;
 import org.apache.spark.sql.SparkSession;
 import org.apache.spark.sql.catalyst.catalog.CatalogTable;
 import org.apache.spark.sql.catalyst.catalog.CatalogUtils;
@@ -173,10 +174,9 @@ protected void ensureNameMappingPresent(Table table) {
   }
 
   protected String getMetadataLocation(Table table) {
-    return table
-        .properties()
-        .getOrDefault(
-            TableProperties.WRITE_METADATA_LOCATION,
-            table.location() + "/" + ICEBERG_METADATA_FOLDER);
+    String defaultValue =
+        LocationUtil.stripTrailingSlash(table.location()) + "/" + ICEBERG_METADATA_FOLDER;
+    return LocationUtil.stripTrailingSlash(
+        table.properties().getOrDefault(TableProperties.WRITE_METADATA_LOCATION, defaultValue));
   }
 }
diff --git a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/procedures/AddFilesProcedure.java b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/procedures/AddFilesProcedure.java
@@ -39,6 +39,7 @@
 import org.apache.iceberg.spark.Spark3Util;
 import org.apache.iceberg.spark.SparkTableUtil;
 import org.apache.iceberg.spark.SparkTableUtil.SparkPartition;
+import org.apache.iceberg.util.LocationUtil;
 import org.apache.spark.sql.catalyst.InternalRow;
 import org.apache.spark.sql.catalyst.TableIdentifier;
 import org.apache.spark.sql.connector.catalog.CatalogPlugin;
@@ -218,8 +219,9 @@ private void importPartitions(
   }
 
   private String getMetadataLocation(Table table) {
-    String defaultValue = table.location() + "/metadata";
-    return table.properties().getOrDefault(TableProperties.WRITE_METADATA_LOCATION, defaultValue);
+    String defaultValue = LocationUtil.stripTrailingSlash(table.location()) + "/metadata";
+    return LocationUtil.stripTrailingSlash(
+        table.properties().getOrDefault(TableProperties.WRITE_METADATA_LOCATION, defaultValue));
   }
 
   @Override