diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/actions/DeletionVectorDescriptor.scala b/spark/src/main/scala/org/apache/spark/sql/delta/actions/DeletionVectorDescriptor.scala index 50f8c861a2c..fc31885993f 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/actions/DeletionVectorDescriptor.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/actions/DeletionVectorDescriptor.scala @@ -21,6 +21,7 @@ import java.net.URI import java.util.{Base64, UUID} import org.apache.spark.sql.delta.DeltaErrors +import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.{Codec, DeltaEncoder, JsonUtils} import com.fasterxml.jackson.annotation.JsonIgnore import com.fasterxml.jackson.databind.annotation.JsonDeserialize @@ -29,6 +30,7 @@ import org.apache.hadoop.fs.Path import org.apache.spark.paths.SparkPath import org.apache.spark.sql.{Column, Encoder} import org.apache.spark.sql.functions.{concat, lit, when} +import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ import org.apache.spark.util.Utils @@ -202,7 +204,8 @@ case class DeletionVectorDescriptor( object DeletionVectorDescriptor { /** Prefix that is used in all file names generated by deletion vector store. */ - val DELETION_VECTOR_FILE_NAME_PREFIX = if (Utils.isTesting) "test%dv%prefix-" else "" + val DELETION_VECTOR_FILE_NAME_PREFIX = SQLConf.get.getConf(DeltaSQLConf.TEST_DV_NAME_PREFIX) + /** String that is used in all file names generated by deletion vector store */ val DELETION_VECTOR_FILE_NAME_CORE = DELETION_VECTOR_FILE_NAME_PREFIX + "deletion_vector" diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/files/DelayedCommitProtocol.scala b/spark/src/main/scala/org/apache/spark/sql/delta/files/DelayedCommitProtocol.scala index a7220fba98b..b41aaa545a2 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/files/DelayedCommitProtocol.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/files/DelayedCommitProtocol.scala @@ -35,6 +35,7 @@ import org.apache.spark.internal.io.FileCommitProtocol.TaskCommitMessage import org.apache.spark.sql.catalyst.expressions.Cast import org.apache.spark.sql.delta.files.DeltaFileFormatWriter.PartitionedTaskAttemptContextImpl import org.apache.spark.sql.delta.sources.DeltaSQLConf +import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{DataType, StringType, TimestampType} import org.apache.spark.util.Utils @@ -110,7 +111,7 @@ class DelayedCommitProtocol( } /** Prefix added in testing mode to all filenames to test special chars that need URL-encoding. */ - val FILE_NAME_PREFIX = if (Utils.isTesting) "test%file%prefix-" else "" + val FILE_NAME_PREFIX = SQLConf.get.getConf(DeltaSQLConf.TEST_FILE_NAME_PREFIX) protected def getFileName( taskContext: TaskAttemptContext, diff --git a/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala b/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala index dc61080b106..f9fbb1167c8 100644 --- a/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala +++ b/spark/src/main/scala/org/apache/spark/sql/delta/sources/DeltaSQLConf.scala @@ -2026,6 +2026,20 @@ trait DeltaSQLConfBase { .booleanConf .createWithDefault(Utils.isTesting) + val TEST_FILE_NAME_PREFIX = + buildStaticConf("testOnly.dataFileNamePrefix") + .internal() + .doc("[TEST_ONLY]: The prefix to use for the names of all Parquet data files.") + .stringConf + .createWithDefault(if (Utils.isTesting) "test%file%prefix-" else "") + + val TEST_DV_NAME_PREFIX = + buildStaticConf("testOnly.dvFileNamePrefix") + .internal() + .doc("[TEST_ONLY]: The prefix to use for the names of all Deletion Vector files.") + .stringConf + .createWithDefault(if (Utils.isTesting) "test%dv%prefix-" else "") + /////////// // UTC TIMESTAMP PARTITION VALUES ///////////////////