Skip to content

Commit f0ffe0c

Browse files
yaooqinncloud-fan
authored andcommitted
[SPARK-33992][SQL] override transformUpWithNewOutput to add allowInvokingTransformsInAnalyzer
### What changes were proposed in this pull request? In #29643, we move the plan rewriting methods to QueryPlan. we need to override transformUpWithNewOutput to add allowInvokingTransformsInAnalyzer because it and resolveOperatorsUpWithNewOutput are called in the analyzer. For example, PaddingAndLengthCheckForCharVarchar could fail query when resolveOperatorsUpWithNewOutput with ```logtalk [info] - char/varchar resolution in sub query *** FAILED *** (367 milliseconds) [info] java.lang.RuntimeException: This method should not be called in the analyzer [info] at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.assertNotAnalysisRule(AnalysisHelper.scala:150) [info] at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.assertNotAnalysisRule$(AnalysisHelper.scala:146) [info] at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.assertNotAnalysisRule(LogicalPlan.scala:29) [info] at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDown(AnalysisHelper.scala:161) [info] at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDown$(AnalysisHelper.scala:160) [info] at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDown(LogicalPlan.scala:29) [info] at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDown(LogicalPlan.scala:29) [info] at org.apache.spark.sql.catalyst.plans.QueryPlan.org$apache$spark$sql$catalyst$plans$QueryPlan$$updateOuterReferencesInSubquery(QueryPlan.scala:267) ``` ### Why are the changes needed? trivial bugfix ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? new tests Closes #31013 from yaooqinn/SPARK-33992. Authored-by: Kent Yao <yao@apache.org> Signed-off-by: Wenchen Fan <wenchen@databricks.com>
1 parent 15a863f commit f0ffe0c

File tree

2 files changed

+24
-0
lines changed

2 files changed

+24
-0
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AnalysisHelper.scala

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,15 @@ trait AnalysisHelper extends QueryPlan[LogicalPlan] { self: LogicalPlan =>
133133
}
134134
}
135135

136+
override def transformUpWithNewOutput(
137+
rule: PartialFunction[LogicalPlan, (LogicalPlan, Seq[(Attribute, Attribute)])],
138+
skipCond: LogicalPlan => Boolean,
139+
canGetOutput: LogicalPlan => Boolean): LogicalPlan = {
140+
AnalysisHelper.allowInvokingTransformsInAnalyzer {
141+
super.transformUpWithNewOutput(rule, skipCond, canGetOutput)
142+
}
143+
}
144+
136145
/**
137146
* Recursively transforms the expressions of a tree, skipping nodes that have already
138147
* been analyzed.

sql/core/src/test/scala/org/apache/spark/sql/CharVarcharTestSuite.scala

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -451,6 +451,21 @@ trait CharVarcharTestSuite extends QueryTest with SQLTestUtils {
451451
Seq(Row("char(5)"), Row("varchar(3)")))
452452
}
453453
}
454+
455+
test("SPARK-33992: char/varchar resolution in correlated sub query") {
456+
withTable("t1", "t2") {
457+
sql(s"CREATE TABLE t1(v VARCHAR(3), c CHAR(5)) USING $format")
458+
sql(s"CREATE TABLE t2(v VARCHAR(3), c CHAR(5)) USING $format")
459+
sql("INSERT INTO t1 VALUES ('c', 'b')")
460+
sql("INSERT INTO t2 VALUES ('a', 'b')")
461+
462+
checkAnswer(sql(
463+
"""
464+
|SELECT v FROM t1
465+
|WHERE 'a' IN (SELECT v FROM t2 WHERE t1.c = t2.c )""".stripMargin),
466+
Row("c"))
467+
}
468+
}
454469
}
455470

456471
// Some basic char/varchar tests which doesn't rely on table implementation.

0 commit comments

Comments
 (0)