Skip to content

Commit 2fe7ec4

Browse files
committed
address comments
1 parent 18907cb commit 2fe7ec4

File tree

1 file changed

+4
-0
lines changed

1 file changed

+4
-0
lines changed

sql/core/src/main/scala/org/apache/spark/sql/execution/command/createDataSourceTables.scala

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,10 @@ case class CreateDataSourceTableCommand(table: CatalogTable, ignoreIfExists: Boo
9494
// empty schema in metastore and infer it at runtime. Note that this also means the new
9595
// scalable partitioning handling feature(introduced at Spark 2.1) is disabled in this case.
9696
case r: HadoopFsRelation if r.overlappedPartCols.nonEmpty =>
97+
logWarning("It is not recommended to create a table with overlapped data and partition " +
98+
"columns, as Spark cannot store a valid table schema and has to infer it at runtime, " +
99+
"which hurts performance. Please check your data files and remove the partition " +
100+
"columns in it.")
97101
table.copy(schema = new StructType(), partitionColumnNames = Nil)
98102

99103
case _ =>

0 commit comments

Comments
 (0)