Skip to content

Commit 5aba2b3

Browse files
MaxGekkMaxGekk
committed
[SPARK-38949][SQL] Wrap SQL statements by double quotes in error messages
### What changes were proposed in this pull request? In the PR, I propose to wrap any SQL statement in error messages by double quotes "", and apply new implementation of `QueryErrorsBase.toSQLStmt()` to all exceptions in `Query.*Errors` w/ error classes. Also this PR modifies all affected tests, see the list in the section "How was this patch tested?". ### Why are the changes needed? To improve user experience with Spark SQL by highlighting SQL statements in error massage and make them more visible to users. ### Does this PR introduce _any_ user-facing change? Yes. The changes might influence on error messages that are visible to users. Before: ```sql The operation DESC PARTITION is not allowed ``` After: ```sql The operation "DESC PARTITION" is not allowed ``` ### How was this patch tested? By running affected test suites: ``` $ build/sbt "sql/testOnly *QueryExecutionErrorsSuite" $ build/sbt "sql/testOnly *QueryParsingErrorsSuite" $ build/sbt "sql/testOnly *QueryCompilationErrorsSuite" $ build/sbt "test:testOnly *QueryCompilationErrorsDSv2Suite" $ build/sbt "test:testOnly *ExtractPythonUDFFromJoinConditionSuite" $ build/sbt "testOnly *PlanParserSuite" $ build/sbt "sql/testOnly *SQLQueryTestSuite -- -z transform.sql" $ build/sbt "sql/testOnly *SQLQueryTestSuite -- -z join-lateral.sql" $ build/sbt "sql/testOnly *SQLQueryTestSuite -- -z describe.sql" ``` Closes #36259 from MaxGekk/error-class-apply-toSQLStmt. Authored-by: Max Gekk <max.gekk@gmail.com> Signed-off-by: Max Gekk <max.gekk@gmail.com>
1 parent 8895ac2 commit 5aba2b3

File tree

17 files changed

+114
-81
lines changed

17 files changed

+114
-81
lines changed

‎python/pyspark/sql/tests/test_udf.py

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -258,15 +258,16 @@ def test_udf_not_supported_in_join_condition(self):
258258
def runWithJoinType(join_type, type_string):
259259
with self.assertRaisesRegex(
260260
AnalysisException,
261-
"Using PythonUDF in join condition of join type %s is not supported" % type_string,
261+
"""Using PythonUDF in join condition of join type "%s" is not supported"""
262+
% type_string,
262263
):
263264
left.join(right, [f("a", "b"), left.a1 == right.b1], join_type).collect()
264265

265-
runWithJoinType("full", "FullOuter")
266-
runWithJoinType("left", "LeftOuter")
267-
runWithJoinType("right", "RightOuter")
268-
runWithJoinType("leftanti", "LeftAnti")
269-
runWithJoinType("leftsemi", "LeftSemi")
266+
runWithJoinType("full", "FULL OUTER")
267+
runWithJoinType("left", "LEFT OUTER")
268+
runWithJoinType("right", "RIGHT OUTER")
269+
runWithJoinType("leftanti", "LEFT ANTI")
270+
runWithJoinType("leftsemi", "LEFT SEMI")
270271

271272
def test_udf_as_join_condition(self):
272273
left = self.spark.createDataFrame([Row(a=1, a1=1, a2=1), Row(a=2, a1=2, a2=2)])

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1166,7 +1166,7 @@ class AstBuilder extends SqlBaseParserBaseVisitor[AnyRef] with SQLConfHelper wit
11661166
}
11671167
if (join.LATERAL != null) {
11681168
if (!Seq(Inner, Cross, LeftOuter).contains(joinType)) {
1169-
throw QueryParsingErrors.unsupportedLateralJoinTypeError(ctx, joinType.toString)
1169+
throw QueryParsingErrors.unsupportedLateralJoinTypeError(ctx, joinType.sql)
11701170
}
11711171
LateralJoin(left, LateralSubquery(plan(join.right)), joinType, condition)
11721172
} else {

‎sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,8 @@ object QueryCompilationErrors extends QueryErrorsBase {
9595
new AnalysisException(
9696
errorClass = "UNSUPPORTED_FEATURE",
9797
messageParameters = Array(
98-
s"IF NOT EXISTS for the table ${toSQLId(tableName)} by INSERT INTO."))
98+
s"${toSQLStmt("IF NOT EXISTS")} for the table ${toSQLId(tableName)} " +
99+
s"by ${toSQLStmt("INSERT INTO")}."))
99100
}
100101

101102
def nonPartitionColError(partitionName: String): Throwable = {
@@ -1573,7 +1574,8 @@ object QueryCompilationErrors extends QueryErrorsBase {
15731574
new AnalysisException(
15741575
errorClass = "UNSUPPORTED_FEATURE",
15751576
messageParameters = Array(
1576-
s"Using PythonUDF in join condition of join type $joinType is not supported"))
1577+
"Using PythonUDF in join condition of join type " +
1578+
s"${toSQLStmt(joinType.sql)} is not supported."))
15771579
}
15781580

15791581
def conflictingAttributesInJoinConditionError(

‎sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryErrorsBase.scala

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717

1818
package org.apache.spark.sql.errors
1919

20+
import java.util.Locale
21+
2022
import org.apache.spark.sql.catalyst.expressions.Literal
2123
import org.apache.spark.sql.catalyst.util.quoteIdentifier
2224
import org.apache.spark.sql.types.{DataType, DoubleType, FloatType}
@@ -48,7 +50,7 @@ trait QueryErrorsBase {
4850

4951
// Quote sql statements in error messages.
5052
def toSQLStmt(text: String): String = {
51-
s"'$text'"
53+
"\"" + text.toUpperCase(Locale.ROOT) + "\""
5254
}
5355

5456
def toSQLId(parts: Seq[String]): String = {

‎sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1892,13 +1892,13 @@ object QueryExecutionErrors extends QueryErrorsBase {
18921892
def repeatedPivotsUnsupportedError(): Throwable = {
18931893
new SparkUnsupportedOperationException(
18941894
errorClass = "UNSUPPORTED_FEATURE",
1895-
messageParameters = Array("Repeated pivots."))
1895+
messageParameters = Array(s"Repeated ${toSQLStmt("pivot")}s."))
18961896
}
18971897

18981898
def pivotNotAfterGroupByUnsupportedError(): Throwable = {
18991899
new SparkUnsupportedOperationException(
19001900
errorClass = "UNSUPPORTED_FEATURE",
1901-
messageParameters = Array("Pivot not after a groupBy."))
1901+
messageParameters = Array(s"${toSQLStmt("pivot")} not after a ${toSQLStmt("group by")}."))
19021902
}
19031903

19041904
private val aesFuncName = toSQLId("aes_encrypt") + "/" + toSQLId("aes_decrypt")

‎sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryParsingErrors.scala

Lines changed: 52 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -90,33 +90,51 @@ object QueryParsingErrors extends QueryErrorsBase {
9090
}
9191

9292
def transformNotSupportQuantifierError(ctx: ParserRuleContext): Throwable = {
93-
new ParseException("UNSUPPORTED_FEATURE",
94-
Array("TRANSFORM does not support DISTINCT/ALL in inputs"), ctx)
93+
new ParseException(
94+
errorClass = "UNSUPPORTED_FEATURE",
95+
messageParameters = Array(s"${toSQLStmt("TRANSFORM")} does not support" +
96+
s" ${toSQLStmt("DISTINCT")}/${toSQLStmt("ALL")} in inputs"),
97+
ctx)
9598
}
9699

97100
def transformWithSerdeUnsupportedError(ctx: ParserRuleContext): Throwable = {
98-
new ParseException("UNSUPPORTED_FEATURE",
99-
Array("TRANSFORM with serde is only supported in hive mode"), ctx)
101+
new ParseException(
102+
errorClass = "UNSUPPORTED_FEATURE",
103+
messageParameters = Array(
104+
s"${toSQLStmt("TRANSFORM")} with serde is only supported in hive mode"),
105+
ctx)
100106
}
101107

102108
def lateralWithPivotInFromClauseNotAllowedError(ctx: FromClauseContext): Throwable = {
103109
new ParseException("LATERAL cannot be used together with PIVOT in FROM clause", ctx)
104110
}
105111

106112
def lateralJoinWithNaturalJoinUnsupportedError(ctx: ParserRuleContext): Throwable = {
107-
new ParseException("UNSUPPORTED_FEATURE", Array("LATERAL join with NATURAL join."), ctx)
113+
new ParseException(
114+
errorClass = "UNSUPPORTED_FEATURE",
115+
messageParameters = Array(s"${toSQLStmt("LATERAL")} join with ${toSQLStmt("NATURAL")} join."),
116+
ctx)
108117
}
109118

110119
def lateralJoinWithUsingJoinUnsupportedError(ctx: ParserRuleContext): Throwable = {
111-
new ParseException("UNSUPPORTED_FEATURE", Array("LATERAL join with USING join."), ctx)
120+
new ParseException(
121+
errorClass = "UNSUPPORTED_FEATURE",
122+
messageParameters = Array(s"${toSQLStmt("LATERAL")} join with ${toSQLStmt("USING")} join."),
123+
ctx)
112124
}
113125

114126
def unsupportedLateralJoinTypeError(ctx: ParserRuleContext, joinType: String): Throwable = {
115-
new ParseException("UNSUPPORTED_FEATURE", Array(s"LATERAL join type '$joinType'."), ctx)
127+
new ParseException(
128+
errorClass = "UNSUPPORTED_FEATURE",
129+
messageParameters = Array(s"${toSQLStmt("LATERAL")} join type ${toSQLStmt(joinType)}."),
130+
ctx)
116131
}
117132

118133
def invalidLateralJoinRelationError(ctx: RelationPrimaryContext): Throwable = {
119-
new ParseException("INVALID_SQL_SYNTAX", Array("LATERAL can only be used with subquery."), ctx)
134+
new ParseException(
135+
errorClass = "INVALID_SQL_SYNTAX",
136+
messageParameters = Array(s"${toSQLStmt("LATERAL")} can only be used with subquery."),
137+
ctx)
120138
}
121139

122140
def repetitiveWindowDefinitionError(name: String, ctx: WindowClauseContext): Throwable = {
@@ -135,7 +153,7 @@ object QueryParsingErrors extends QueryErrorsBase {
135153
}
136154

137155
def naturalCrossJoinUnsupportedError(ctx: RelationContext): Throwable = {
138-
new ParseException("UNSUPPORTED_FEATURE", Array("NATURAL CROSS JOIN."), ctx)
156+
new ParseException("UNSUPPORTED_FEATURE", Array(toSQLStmt("NATURAL CROSS JOIN") + "."), ctx)
139157
}
140158

141159
def emptyInputForTableSampleError(ctx: ParserRuleContext): Throwable = {
@@ -301,15 +319,18 @@ object QueryParsingErrors extends QueryErrorsBase {
301319
}
302320

303321
def showFunctionsUnsupportedError(identifier: String, ctx: IdentifierContext): Throwable = {
304-
new ParseException("INVALID_SQL_SYNTAX",
305-
Array(s"SHOW ${toSQLId(identifier)} FUNCTIONS not supported"), ctx)
322+
new ParseException(
323+
errorClass = "INVALID_SQL_SYNTAX",
324+
messageParameters = Array(
325+
s"${toSQLStmt("SHOW")} ${toSQLId(identifier)} ${toSQLStmt("FUNCTIONS")} not supported"),
326+
ctx)
306327
}
307328

308329
def showFunctionsInvalidPatternError(pattern: String, ctx: ParserRuleContext): Throwable = {
309330
new ParseException(
310331
errorClass = "INVALID_SQL_SYNTAX",
311332
messageParameters = Array(
312-
s"Invalid pattern in SHOW FUNCTIONS: ${toSQLId(pattern)}. " +
333+
s"Invalid pattern in ${toSQLStmt("SHOW FUNCTIONS")}: ${toSQLId(pattern)}. " +
313334
s"It must be a ${toSQLType(StringType)} literal."),
314335
ctx)
315336
}
@@ -416,13 +437,20 @@ object QueryParsingErrors extends QueryErrorsBase {
416437
}
417438

418439
def createFuncWithBothIfNotExistsAndReplaceError(ctx: CreateFunctionContext): Throwable = {
419-
new ParseException("INVALID_SQL_SYNTAX",
420-
Array("CREATE FUNCTION with both IF NOT EXISTS and REPLACE is not allowed."), ctx)
440+
new ParseException(
441+
errorClass = "INVALID_SQL_SYNTAX",
442+
messageParameters = Array(
443+
s"${toSQLStmt("CREATE FUNCTION")} with both ${toSQLStmt("IF NOT EXISTS")} " +
444+
s"and ${toSQLStmt("REPLACE")} is not allowed."),
445+
ctx)
421446
}
422447

423448
def defineTempFuncWithIfNotExistsError(ctx: CreateFunctionContext): Throwable = {
424-
new ParseException("INVALID_SQL_SYNTAX",
425-
Array("It is not allowed to define a TEMPORARY function with IF NOT EXISTS."), ctx)
449+
new ParseException(
450+
errorClass = "INVALID_SQL_SYNTAX",
451+
messageParameters = Array(
452+
s"It is not allowed to define a ${toSQLStmt("TEMPORARY FUNCTION")}" +
453+
s" with ${toSQLStmt("IF NOT EXISTS")}."), ctx)
426454
}
427455

428456
def unsupportedFunctionNameError(funcName: Seq[String], ctx: CreateFunctionContext): Throwable = {
@@ -435,7 +463,8 @@ object QueryParsingErrors extends QueryErrorsBase {
435463
ctx: CreateFunctionContext): Throwable = {
436464
new ParseException(
437465
"INVALID_SQL_SYNTAX",
438-
Array("Specifying a database in CREATE TEMPORARY FUNCTION is not allowed: " +
466+
Array(
467+
s"Specifying a database in ${toSQLStmt("CREATE TEMPORARY FUNCTION")} is not allowed: " +
439468
toSQLId(databaseName)),
440469
ctx)
441470
}
@@ -449,8 +478,12 @@ object QueryParsingErrors extends QueryErrorsBase {
449478
}
450479

451480
def invalidNameForDropTempFunc(name: Seq[String], ctx: ParserRuleContext): Throwable = {
452-
new ParseException("INVALID_SQL_SYNTAX",
453-
Array(s"DROP TEMPORARY FUNCTION requires a single part name but got: ${toSQLId(name)}"), ctx)
481+
new ParseException(
482+
errorClass = "INVALID_SQL_SYNTAX",
483+
messageParameters = Array(
484+
s"${toSQLStmt("DROP TEMPORARY FUNCTION")} requires a single part name but got: " +
485+
toSQLId(name)),
486+
ctx)
454487
}
455488

456489
def defaultColumnNotImplementedYetError(ctx: ParserRuleContext): Throwable = {

‎sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ExtractPythonUDFFromJoinConditionSuite.scala

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -187,9 +187,9 @@ class ExtractPythonUDFFromJoinConditionSuite extends PlanTest {
187187
condition = Some(unevaluableJoinCond))
188188
Optimize.execute(query.analyze)
189189
}
190-
assert(e.message.contentEquals(
190+
assert(e.message ==
191191
"The feature is not supported: " +
192-
s"Using PythonUDF in join condition of join type $joinType is not supported"))
192+
s"""Using PythonUDF in join condition of join type "${joinType.sql}" is not supported.""")
193193

194194
val query2 = testRelationLeft.join(
195195
testRelationRight,

‎sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/PlanParserSuite.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1250,7 +1250,7 @@ class PlanParserSuite extends AnalysisTest {
12501250
| "escapeChar" = "\\")
12511251
|FROM testData
12521252
""".stripMargin,
1253-
"TRANSFORM with serde is only supported in hive mode")
1253+
"\"TRANSFORM\" with serde is only supported in hive mode")
12541254
}
12551255

12561256

‎sql/core/src/test/resources/sql-tests/results/describe.sql.out

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -462,7 +462,7 @@ DESC temp_v PARTITION (c='Us', d=1)
462462
struct<>
463463
-- !query output
464464
org.apache.spark.sql.AnalysisException
465-
The operation 'DESC PARTITION' is not allowed on the temporary view: `temp_v`
465+
The operation "DESC PARTITION" is not allowed on the temporary view: `temp_v`
466466

467467

468468
-- !query
@@ -539,7 +539,7 @@ DESC v PARTITION (c='Us', d=1)
539539
struct<>
540540
-- !query output
541541
org.apache.spark.sql.AnalysisException
542-
The operation 'DESC PARTITION' is not allowed on the view: `v`
542+
The operation "DESC PARTITION" is not allowed on the view: `v`
543543

544544

545545
-- !query

‎sql/core/src/test/resources/sql-tests/results/join-lateral.sql.out

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,7 @@ struct<>
153153
-- !query output
154154
org.apache.spark.sql.catalyst.parser.ParseException
155155

156-
The feature is not supported: LATERAL join with NATURAL join.(line 1, pos 14)
156+
The feature is not supported: "LATERAL" join with "NATURAL" join.(line 1, pos 14)
157157

158158
== SQL ==
159159
SELECT * FROM t1 NATURAL JOIN LATERAL (SELECT c1 + c2 AS c2)
@@ -167,7 +167,7 @@ struct<>
167167
-- !query output
168168
org.apache.spark.sql.catalyst.parser.ParseException
169169

170-
The feature is not supported: LATERAL join with USING join.(line 1, pos 14)
170+
The feature is not supported: "LATERAL" join with "USING" join.(line 1, pos 14)
171171

172172
== SQL ==
173173
SELECT * FROM t1 JOIN LATERAL (SELECT c1 + c2 AS c2) USING (c2)

‎sql/core/src/test/resources/sql-tests/results/transform.sql.out

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -719,7 +719,7 @@ struct<>
719719
-- !query output
720720
org.apache.spark.sql.catalyst.parser.ParseException
721721

722-
The feature is not supported: TRANSFORM does not support DISTINCT/ALL in inputs(line 1, pos 17)
722+
The feature is not supported: "TRANSFORM" does not support "DISTINCT"/"ALL" in inputs(line 1, pos 17)
723723

724724
== SQL ==
725725
SELECT TRANSFORM(DISTINCT b, a, c)
@@ -739,7 +739,7 @@ struct<>
739739
-- !query output
740740
org.apache.spark.sql.catalyst.parser.ParseException
741741

742-
The feature is not supported: TRANSFORM does not support DISTINCT/ALL in inputs(line 1, pos 17)
742+
The feature is not supported: "TRANSFORM" does not support "DISTINCT"/"ALL" in inputs(line 1, pos 17)
743743

744744
== SQL ==
745745
SELECT TRANSFORM(ALL b, a, c)

‎sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsDSv2Suite.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ class QueryCompilationErrorsDSv2Suite
5252

5353
checkAnswer(spark.table(tbl), spark.emptyDataFrame)
5454
assert(e.getMessage === "The feature is not supported: " +
55-
s"IF NOT EXISTS for the table `testcat`.`ns1`.`ns2`.`tbl` by INSERT INTO.")
55+
s""""IF NOT EXISTS" for the table `testcat`.`ns1`.`ns2`.`tbl` by "INSERT INTO".""")
5656
assert(e.getErrorClass === "UNSUPPORTED_FEATURE")
5757
assert(e.getSqlState === "0A000")
5858
}

‎sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -155,7 +155,7 @@ class QueryCompilationErrorsSuite extends QueryTest with SharedSparkSession {
155155
assert(e.getSqlState === "0A000")
156156
assert(e.message ===
157157
"The feature is not supported: " +
158-
"Using PythonUDF in join condition of join type LeftOuter is not supported")
158+
"Using PythonUDF in join condition of join type \"LEFT OUTER\" is not supported.")
159159
}
160160

161161
test("UNSUPPORTED_FEATURE: Using pandas UDF aggregate expression with pivot") {
@@ -333,7 +333,8 @@ class QueryCompilationErrorsSuite extends QueryTest with SharedSparkSession {
333333
)
334334
assert(e.getErrorClass === "FORBIDDEN_OPERATION")
335335
assert(e.message ===
336-
s"The operation 'DESC PARTITION' is not allowed on the temporary view: `$tempViewName`")
336+
s"""The operation "DESC PARTITION" is not allowed """ +
337+
s"on the temporary view: `$tempViewName`")
337338
}
338339
}
339340
}
@@ -358,7 +359,8 @@ class QueryCompilationErrorsSuite extends QueryTest with SharedSparkSession {
358359
)
359360
assert(e.getErrorClass === "FORBIDDEN_OPERATION")
360361
assert(e.message ===
361-
s"The operation 'DESC PARTITION' is not allowed on the view: `$viewName`")
362+
s"""The operation "DESC PARTITION" is not allowed """ +
363+
s"on the view: `$viewName`")
362364
}
363365
}
364366
}

‎sql/core/src/test/scala/org/apache/spark/sql/errors/QueryExecutionErrorsSuite.scala

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -163,7 +163,7 @@ class QueryExecutionErrorsSuite extends QueryTest
163163
}
164164
assert(e1.getErrorClass === "UNSUPPORTED_FEATURE")
165165
assert(e1.getSqlState === "0A000")
166-
assert(e1.getMessage === "The feature is not supported: Repeated pivots.")
166+
assert(e1.getMessage === """The feature is not supported: Repeated "PIVOT"s.""")
167167

168168
val e2 = intercept[SparkUnsupportedOperationException] {
169169
trainingSales
@@ -174,7 +174,7 @@ class QueryExecutionErrorsSuite extends QueryTest
174174
}
175175
assert(e2.getErrorClass === "UNSUPPORTED_FEATURE")
176176
assert(e2.getSqlState === "0A000")
177-
assert(e2.getMessage === "The feature is not supported: Pivot not after a groupBy.")
177+
assert(e2.getMessage === """The feature is not supported: "PIVOT" not after a "GROUP BY".""")
178178
}
179179

180180
test("INCONSISTENT_BEHAVIOR_CROSS_VERSION: " +

0 commit comments

Comments
 (0)