From 6887df3dca42645ab950f1c5565240c21617200d Mon Sep 17 00:00:00 2001 From: Prashant Singh <35593236+singhpk234@users.noreply.github.com> Date: Sat, 5 Nov 2022 11:59:37 -0700 Subject: [PATCH] Spark: Strip trailing slash from metadatalocation (#6121) Co-authored-by: Prashant Singh --- .../apache/iceberg/BaseMetastoreTableOperations.java | 3 ++- .../org/apache/iceberg/rest/RESTTableOperations.java | 3 ++- .../spark/actions/BaseTableCreationSparkAction.java | 10 +++++----- .../iceberg/spark/procedures/AddFilesProcedure.java | 6 ++++-- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/BaseMetastoreTableOperations.java b/core/src/main/java/org/apache/iceberg/BaseMetastoreTableOperations.java index e66970c2d547..34d1ac2cbb3f 100644 --- a/core/src/main/java/org/apache/iceberg/BaseMetastoreTableOperations.java +++ b/core/src/main/java/org/apache/iceberg/BaseMetastoreTableOperations.java @@ -43,6 +43,7 @@ import org.apache.iceberg.relocated.com.google.common.base.Objects; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.apache.iceberg.util.LocationUtil; import org.apache.iceberg.util.PropertyUtil; import org.apache.iceberg.util.Tasks; import org.slf4j.Logger; @@ -220,7 +221,7 @@ private String metadataFileLocation(TableMetadata metadata, String filename) { String metadataLocation = metadata.properties().get(TableProperties.WRITE_METADATA_LOCATION); if (metadataLocation != null) { - return String.format("%s/%s", metadataLocation, filename); + return String.format("%s/%s", LocationUtil.stripTrailingSlash(metadataLocation), filename); } else { return String.format("%s/%s/%s", metadata.location(), METADATA_FOLDER_NAME, filename); } diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTTableOperations.java b/core/src/main/java/org/apache/iceberg/rest/RESTTableOperations.java index 639856eb2dca..0841de763bf1 100644 --- a/core/src/main/java/org/apache/iceberg/rest/RESTTableOperations.java +++ b/core/src/main/java/org/apache/iceberg/rest/RESTTableOperations.java @@ -37,6 +37,7 @@ import org.apache.iceberg.rest.requests.UpdateTableRequest; import org.apache.iceberg.rest.responses.ErrorResponse; import org.apache.iceberg.rest.responses.LoadTableResponse; +import org.apache.iceberg.util.LocationUtil; class RESTTableOperations implements TableOperations { private static final String METADATA_FOLDER_NAME = "metadata"; @@ -169,7 +170,7 @@ private static String metadataFileLocation(TableMetadata metadata, String filena String metadataLocation = metadata.properties().get(TableProperties.WRITE_METADATA_LOCATION); if (metadataLocation != null) { - return String.format("%s/%s", metadataLocation, filename); + return String.format("%s/%s", LocationUtil.stripTrailingSlash(metadataLocation), filename); } else { return String.format("%s/%s/%s", metadata.location(), METADATA_FOLDER_NAME, filename); } diff --git a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/actions/BaseTableCreationSparkAction.java b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/actions/BaseTableCreationSparkAction.java index 9639b205ac59..520c520484dc 100644 --- a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/actions/BaseTableCreationSparkAction.java +++ b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/actions/BaseTableCreationSparkAction.java @@ -37,6 +37,7 @@ import org.apache.iceberg.spark.SparkCatalog; import org.apache.iceberg.spark.SparkSessionCatalog; import org.apache.iceberg.spark.source.StagedSparkTable; +import org.apache.iceberg.util.LocationUtil; import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.catalyst.catalog.CatalogTable; import org.apache.spark.sql.catalyst.catalog.CatalogUtils; @@ -173,10 +174,9 @@ protected void ensureNameMappingPresent(Table table) { } protected String getMetadataLocation(Table table) { - return table - .properties() - .getOrDefault( - TableProperties.WRITE_METADATA_LOCATION, - table.location() + "/" + ICEBERG_METADATA_FOLDER); + String defaultValue = + LocationUtil.stripTrailingSlash(table.location()) + "/" + ICEBERG_METADATA_FOLDER; + return LocationUtil.stripTrailingSlash( + table.properties().getOrDefault(TableProperties.WRITE_METADATA_LOCATION, defaultValue)); } } diff --git a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/procedures/AddFilesProcedure.java b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/procedures/AddFilesProcedure.java index ea0ca4c5a9b9..e1be42e17597 100644 --- a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/procedures/AddFilesProcedure.java +++ b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/procedures/AddFilesProcedure.java @@ -39,6 +39,7 @@ import org.apache.iceberg.spark.Spark3Util; import org.apache.iceberg.spark.SparkTableUtil; import org.apache.iceberg.spark.SparkTableUtil.SparkPartition; +import org.apache.iceberg.util.LocationUtil; import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.sql.catalyst.TableIdentifier; import org.apache.spark.sql.connector.catalog.CatalogPlugin; @@ -218,8 +219,9 @@ private void importPartitions( } private String getMetadataLocation(Table table) { - String defaultValue = table.location() + "/metadata"; - return table.properties().getOrDefault(TableProperties.WRITE_METADATA_LOCATION, defaultValue); + String defaultValue = LocationUtil.stripTrailingSlash(table.location()) + "/metadata"; + return LocationUtil.stripTrailingSlash( + table.properties().getOrDefault(TableProperties.WRITE_METADATA_LOCATION, defaultValue)); } @Override