Skip to content

Commit 3d7a20a

Browse files
wangyumyaooqinn
authored andcommitted
[SPARK-48709][SQL][3.5] Fix varchar type resolution mismatch for DataSourceV2 CTAS
Backport of #47082. ### What changes were proposed in this pull request? This PR fixes varchar type resolution mismatch for DataSourceV2 CTAS. For example: ```sql set spark.sql.storeAssignmentPolicy=LEGACY; CREATE TABLE testcat.ns.t1 (d1 string, d2 varchar(200)) USING parquet; CREATE TABLE testcat.ns.t2 USING foo as select * from testcat.ns.t1 ``` Error message: ``` org.apache.spark.sql.AnalysisException: LEGACY store assignment policy is disallowed in Spark data source V2. Please set the configuration spark.sql.storeAssignmentPolicy to other values. ``` ### Why are the changes needed? Avoid query failures. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Unit test. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #47103 from wangyum/SPARK-48709-branch-3.5. Authored-by: Yuming Wang <yumwang@ebay.com> Signed-off-by: Kent Yao <yao@apache.org>
1 parent b28ddb1 commit 3d7a20a

File tree

2 files changed

+12
-1
lines changed

2 files changed

+12
-1
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,10 +62,11 @@ trait V2WriteCommand extends UnaryCommand with KeepAnalyzedQuery {
6262
table.skipSchemaResolution || (query.output.size == table.output.size &&
6363
query.output.zip(table.output).forall {
6464
case (inAttr, outAttr) =>
65+
val inType = CharVarcharUtils.getRawType(inAttr.metadata).getOrElse(inAttr.dataType)
6566
val outType = CharVarcharUtils.getRawType(outAttr.metadata).getOrElse(outAttr.dataType)
6667
// names and types must match, nullability must be compatible
6768
inAttr.name == outAttr.name &&
68-
DataType.equalsIgnoreCompatibleNullability(inAttr.dataType, outType) &&
69+
DataType.equalsIgnoreCompatibleNullability(inType, outType) &&
6970
(outAttr.nullable || !inAttr.nullable)
7071
})
7172
}

sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1736,6 +1736,16 @@ class DataSourceV2SQLSuiteV1Filter
17361736
}
17371737
}
17381738

1739+
test("SPARK-48709: varchar resolution mismatch for DataSourceV2 CTAS") {
1740+
withSQLConf(
1741+
SQLConf.STORE_ASSIGNMENT_POLICY.key -> SQLConf.StoreAssignmentPolicy.LEGACY.toString) {
1742+
withTable("testcat.ns.t1", "testcat.ns.t2") {
1743+
sql("CREATE TABLE testcat.ns.t1 (d1 string, d2 varchar(200)) USING parquet")
1744+
sql("CREATE TABLE testcat.ns.t2 USING foo as select * from testcat.ns.t1")
1745+
}
1746+
}
1747+
}
1748+
17391749
test("ShowCurrentNamespace: basic tests") {
17401750
def testShowCurrentNamespace(expectedCatalogName: String, expectedNamespace: String): Unit = {
17411751
val schema = new StructType()

0 commit comments

Comments
 (0)