Skip to content

Commit 1dd37ff

Browse files
gengliangwanggatorsmile
authored andcommitted
[SPARK-23490][BACKPORT][SQL] Check storage.locationUri with existing table in CreateTable
Backport apache#20660 to branch 2.3 ===================================== ## What changes were proposed in this pull request? For CreateTable with Append mode, we should check if `storage.locationUri` is the same with existing table in `PreprocessTableCreation` In the current code, there is only a simple exception if the `storage.locationUri` is different with existing table: `org.apache.spark.sql.AnalysisException: Table or view not found:` which can be improved. ## How was this patch tested? Unit test Author: Wang Gengliang <gengliang.wang@databricks.com> Closes apache#20766 from gengliangwang/backport_20660_to_2.3.
1 parent 86ca915 commit 1dd37ff

File tree

2 files changed

+37
-0
lines changed
  • sql/core/src
    • main/scala/org/apache/spark/sql/execution/datasources
    • test/scala/org/apache/spark/sql/execution/command

2 files changed

+37
-0
lines changed

sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala

+8
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,14 @@ case class PreprocessTableCreation(sparkSession: SparkSession) extends Rule[Logi
118118
s"`${existingProvider.getSimpleName}`. It doesn't match the specified format " +
119119
s"`${specifiedProvider.getSimpleName}`.")
120120
}
121+
tableDesc.storage.locationUri match {
122+
case Some(location) if location.getPath != existingTable.location.getPath =>
123+
throw new AnalysisException(
124+
s"The location of the existing table ${tableIdentWithDB.quotedString} is " +
125+
s"`${existingTable.location}`. It doesn't match the specified location " +
126+
s"`${tableDesc.location}`.")
127+
case _ =>
128+
}
121129

122130
if (query.schema.length != existingTable.schema.length) {
123131
throw new AnalysisException(

sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala

+29
Original file line numberDiff line numberDiff line change
@@ -536,6 +536,35 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
536536
}
537537
}
538538

539+
test("create table - append to a non-partitioned table created with different paths") {
540+
import testImplicits._
541+
withTempDir { dir1 =>
542+
withTempDir { dir2 =>
543+
withTable("path_test") {
544+
Seq(1L -> "a").toDF("v1", "v2")
545+
.write
546+
.mode(SaveMode.Append)
547+
.format("json")
548+
.option("path", dir1.getCanonicalPath)
549+
.saveAsTable("path_test")
550+
551+
val ex = intercept[AnalysisException] {
552+
Seq((3L, "c")).toDF("v1", "v2")
553+
.write
554+
.mode(SaveMode.Append)
555+
.format("json")
556+
.option("path", dir2.getCanonicalPath)
557+
.saveAsTable("path_test")
558+
}.getMessage
559+
assert(ex.contains("The location of the existing table `default`.`path_test`"))
560+
561+
checkAnswer(
562+
spark.table("path_test"), Row(1L, "a") :: Nil)
563+
}
564+
}
565+
}
566+
}
567+
539568
test("Refresh table after changing the data source table partitioning") {
540569
import testImplicits._
541570

0 commit comments

Comments
 (0)