Skip to content

Commit 38d245f

Browse files
dilipbiswalroygao94
authored andcommitted
[SPARK-13698][SQL] Fix Analysis Exceptions when Using Backticks in Generate
## What changes were proposed in this pull request? Analysis exception occurs while running the following query. ``` SELECT ints FROM nestedArray LATERAL VIEW explode(a.b) `a` AS `ints` ``` ``` Failed to analyze query: org.apache.spark.sql.AnalysisException: cannot resolve '`ints`' given input columns: [a, `ints`]; line 1 pos 7 'Project ['ints] +- Generate explode(a#0.b), true, false, Some(a), [`ints`apache#8] +- SubqueryAlias nestedarray +- LocalRelation [a#0], [[[[1,2,3]]]] ``` ## How was this patch tested? Added new unit tests in SQLQuerySuite and HiveQlSuite Author: Dilip Biswal <dbiswal@us.ibm.com> Closes apache#11538 from dilipbiswal/SPARK-13698.
1 parent be25436 commit 38d245f

File tree

3 files changed

+45
-2
lines changed

3 files changed

+45
-2
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/CatalystQl.scala

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -899,10 +899,16 @@ https://cwiki.apache.org/confluence/display/Hive/Enhanced+Aggregation%2C+Cube%2C
899899
}
900900

901901
val attributes = clauses.collect {
902-
case Token(a, Nil) => UnresolvedAttribute(a.toLowerCase)
902+
case Token(a, Nil) => UnresolvedAttribute(cleanIdentifier(a.toLowerCase))
903903
}
904904

905-
Generate(generator, join = true, outer = outer, Some(alias.toLowerCase), attributes, child)
905+
Generate(
906+
generator,
907+
join = true,
908+
outer = outer,
909+
Some(cleanIdentifier(alias.toLowerCase)),
910+
attributes,
911+
child)
906912
}
907913

908914
protected def nodeToGenerator(node: ASTNode): Generator = noParseRule("Generator", node)

sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveQlSuite.scala

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -208,4 +208,24 @@ class HiveQlSuite extends SparkFunSuite with BeforeAndAfterAll {
208208
|USING 'cat' AS (`thing1` int, `thing2` string) FROM `default`.`parquet_t1`) AS t
209209
""".stripMargin)
210210
}
211+
212+
test("use backticks in output of Generator") {
213+
val plan = parser.parsePlan(
214+
"""
215+
|SELECT `gentab2`.`gencol2`
216+
|FROM `default`.`src`
217+
|LATERAL VIEW explode(array(array(1, 2, 3))) `gentab1` AS `gencol1`
218+
|LATERAL VIEW explode(`gentab1`.`gencol1`) `gentab2` AS `gencol2`
219+
""".stripMargin)
220+
}
221+
222+
test("use escaped backticks in output of Generator") {
223+
val plan = parser.parsePlan(
224+
"""
225+
|SELECT `gen``tab2`.`gen``col2`
226+
|FROM `default`.`src`
227+
|LATERAL VIEW explode(array(array(1, 2, 3))) `gen``tab1` AS `gen``col1`
228+
|LATERAL VIEW explode(`gen``tab1`.`gen``col1`) `gen``tab2` AS `gen``col2`
229+
""".stripMargin)
230+
}
211231
}

sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -264,6 +264,23 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
264264
checkAnswer(
265265
sql("SELECT ints FROM nestedArray LATERAL VIEW explode(a.b) a AS ints"),
266266
Row(1) :: Row(2) :: Row(3) :: Nil)
267+
268+
checkAnswer(
269+
sql("SELECT `ints` FROM nestedArray LATERAL VIEW explode(a.b) `a` AS `ints`"),
270+
Row(1) :: Row(2) :: Row(3) :: Nil)
271+
272+
checkAnswer(
273+
sql("SELECT `a`.`ints` FROM nestedArray LATERAL VIEW explode(a.b) `a` AS `ints`"),
274+
Row(1) :: Row(2) :: Row(3) :: Nil)
275+
276+
checkAnswer(
277+
sql(
278+
"""
279+
|SELECT `weird``tab`.`weird``col`
280+
|FROM nestedArray
281+
|LATERAL VIEW explode(a.b) `weird``tab` AS `weird``col`
282+
""".stripMargin),
283+
Row(1) :: Row(2) :: Row(3) :: Nil)
267284
}
268285

269286
test("SPARK-4512 Fix attribute reference resolution error when using SORT BY") {

0 commit comments

Comments
 (0)