Skip to content

Commit 3bc83de

Browse files
gatorsmilecloud-fan
authored andcommitted
[SPARK-26307][SQL] Fix CTAS when INSERT a partitioned table using Hive serde
## What changes were proposed in this pull request? This is a Spark 2.3 regression introduced in apache#20521. We should add the partition info for InsertIntoHiveTable in CreateHiveTableAsSelectCommand. Otherwise, we will hit the following error by running the newly added test case: ``` [info] - CTAS: INSERT a partitioned table using Hive serde *** FAILED *** (829 milliseconds) [info] org.apache.spark.SparkException: Requested partitioning does not match the tab1 table: [info] Requested partitions: [info] Table partitions: part [info] at org.apache.spark.sql.hive.execution.InsertIntoHiveTable.processInsert(InsertIntoHiveTable.scala:179) [info] at org.apache.spark.sql.hive.execution.InsertIntoHiveTable.run(InsertIntoHiveTable.scala:107) ``` ## How was this patch tested? Added a test case. Closes apache#23255 from gatorsmile/fixCTAS. Authored-by: gatorsmile <gatorsmile@gmail.com> Signed-off-by: Wenchen Fan <wenchen@databricks.com>
1 parent 403c8d5 commit 3bc83de

File tree

2 files changed

+14
-1
lines changed

2 files changed

+14
-1
lines changed

sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/CreateHiveTableAsSelectCommand.scala

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,9 +57,11 @@ case class CreateHiveTableAsSelectCommand(
5757
return Seq.empty
5858
}
5959

60+
// For CTAS, there is no static partition values to insert.
61+
val partition = tableDesc.partitionColumnNames.map(_ -> None).toMap
6062
InsertIntoHiveTable(
6163
tableDesc,
62-
Map.empty,
64+
partition,
6365
query,
6466
overwrite = false,
6567
ifPartitionNotExists = false,

sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertSuite.scala

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -752,6 +752,17 @@ class InsertSuite extends QueryTest with TestHiveSingleton with BeforeAndAfter
752752
}
753753
}
754754

755+
test("SPARK-26307: CTAS - INSERT a partitioned table using Hive serde") {
756+
withTable("tab1") {
757+
withSQLConf("hive.exec.dynamic.partition.mode" -> "nonstrict") {
758+
val df = Seq(("a", 100)).toDF("part", "id")
759+
df.write.format("hive").partitionBy("part").mode("overwrite").saveAsTable("tab1")
760+
df.write.format("hive").partitionBy("part").mode("append").saveAsTable("tab1")
761+
}
762+
}
763+
}
764+
765+
755766
Seq("LOCAL", "").foreach { local =>
756767
Seq(true, false).foreach { caseSensitivity =>
757768
Seq("orc", "parquet").foreach { format =>

0 commit comments

Comments
 (0)