Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[SPARK-49951][SQL] Assign proper error condition for _LEGACY_ERROR_TEMP_(1099|3085) #48449

Open
wants to merge 6 commits into
base: master
Choose a base branch
from
+44 −46
Open

[SPARK-49951][SQL] Assign proper error condition for _LEGACY_ERROR_TEMP_(1099|3085) #48449

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
51f33da
[SPARK-49951][SQL] Assign proper error condition for _LEGACY_ERROR_TE…
itholic Oct 14, 2024
c14530c
Merge branch 'master' of https://github.com/apache/spark into SPARK-4…
itholic Oct 14, 2024
768a29e
applied comment
itholic Oct 15, 2024
1746d5a
fix test
itholic Oct 15, 2024
07ef019
remove unused import
itholic Oct 15, 2024
046b63a
fix test
itholic Oct 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 6 additions & 10 deletions common/utils/src/main/resources/error/error-conditions.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3802,6 +3802,12 @@
],
"sqlState" : "42617"
},
"PARSE_MODE_UNSUPPORTED" : {
"message" : [
"<funcName>() doesn't support the <mode> mode. Acceptable modes are <permissiveMode> and <failFastMode>."
itholic marked this conversation as resolved.
Show resolved Hide resolved
],
"sqlState" : "42601"
},
"PARSE_SYNTAX_ERROR" : {
"message" : [
"Syntax error at or near <error><hint>."
Expand Down Expand Up @@ -6015,11 +6021,6 @@
"DataType '<x>' is not supported by <className>."
]
},
"_LEGACY_ERROR_TEMP_1099" : {
"message" : [
"<funcName>() doesn't support the <mode> mode. Acceptable modes are <permissiveMode> and <failFastMode>."
]
},
"_LEGACY_ERROR_TEMP_1103" : {
"message" : [
"Unsupported component type <clz> in arrays."
Expand Down Expand Up @@ -8097,11 +8098,6 @@
"No handler for UDF/UDAF/UDTF '<clazz>': <e>"
]
},
"_LEGACY_ERROR_TEMP_3085" : {
"message" : [
"from_avro() doesn't support the <name> mode. Acceptable modes are <permissiveMode> and <failFastMode>."
]
},
"_LEGACY_ERROR_TEMP_3086" : {
"message" : [
"Cannot persist <tableName> into Hive metastore as table property keys may not start with 'spark.sql.': <invalidKeys>"
Expand Down
20 changes: 7 additions & 13 deletions connector/avro/src/main/scala/org/apache/spark/sql/avro/AvroDataToCatalyst.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,10 @@ import org.apache.avro.generic.GenericDatumReader
import org.apache.avro.io.{BinaryDecoder, DecoderFactory}

import org.apache.spark.SparkException
import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.catalyst.expressions.{ExpectsInputTypes, Expression, SpecificInternalRow, UnaryExpression}
import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, CodeGenerator, ExprCode}
import org.apache.spark.sql.catalyst.util.{FailFastMode, ParseMode, PermissiveMode}
import org.apache.spark.sql.errors.QueryCompilationErrors
import org.apache.spark.sql.types._

private[sql] case class AvroDataToCatalyst(
Expand Down Expand Up @@ -80,12 +80,9 @@ private[sql] case class AvroDataToCatalyst(
@transient private lazy val parseMode: ParseMode = {
val mode = avroOptions.parseMode
if (mode != PermissiveMode && mode != FailFastMode) {
throw new AnalysisException(
errorClass = "_LEGACY_ERROR_TEMP_3085",
messageParameters = Map(
"name" -> mode.name,
"permissiveMode" -> PermissiveMode.name,
"failFastMode" -> FailFastMode.name))
throw QueryCompilationErrors.parseModeUnsupportedError(
"from_avro", mode
itholic marked this conversation as resolved.
Show resolved Hide resolved
)
}
mode
}
Expand Down Expand Up @@ -123,12 +120,9 @@ private[sql] case class AvroDataToCatalyst(
s"Current parse Mode: ${FailFastMode.name}. To process malformed records as null " +
"result, try setting the option 'mode' as 'PERMISSIVE'.", e)
case _ =>
throw new AnalysisException(
errorClass = "_LEGACY_ERROR_TEMP_3085",
messageParameters = Map(
"name" -> parseMode.name,
"permissiveMode" -> PermissiveMode.name,
"failFastMode" -> FailFastMode.name))
throw QueryCompilationErrors.parseModeUnsupportedError(
"from_avro", parseMode
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please, use prettyName

)
}
}
}
Expand Down
13 changes: 13 additions & 0 deletions connector/avro/src/test/scala/org/apache/spark/sql/avro/AvroFunctionsSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,19 @@ class AvroFunctionsSuite extends QueryTest with SharedSparkSession {
functions.from_avro(
$"avro", avroTypeStruct, Map("mode" -> "PERMISSIVE").asJava)),
expected)

checkError(
exception = intercept[AnalysisException] {
avroStructDF.select(
functions.from_avro(
$"avro", avroTypeStruct, Map("mode" -> "DROPMALFORMED").asJava)).collect()
},
condition = "PARSE_MODE_UNSUPPORTED",
parameters = Map(
"funcName" -> "from_avro",
"mode" -> "DROPMALFORMED",
"permissiveMode" -> "PERMISSIVE",
"failFastMode" -> "FAILFAST"))
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added new test for _LEGACY_ERROR_TEMP_3085
cc @MaxGekk fyi

}

test("roundtrip in to_avro and from_avro - array with null") {
Expand Down
2 changes: 1 addition & 1 deletion sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1341,7 +1341,7 @@ private[sql] object QueryCompilationErrors extends QueryErrorsBase with Compilat

def parseModeUnsupportedError(funcName: String, mode: ParseMode): Throwable = {
new AnalysisException(
errorClass = "_LEGACY_ERROR_TEMP_1099",
errorClass = "PARSE_MODE_UNSUPPORTED",
messageParameters = Map(
"funcName" -> funcName,
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since it is an id, please, quote it:

Suggested change
"funcName" -> funcName,
"funcName" -> toSQLId(funcName),

"mode" -> mode.name,
Expand Down
2 changes: 1 addition & 1 deletion sql/core/src/test/scala/org/apache/spark/sql/CsvFunctionsSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -352,7 +352,7 @@ class CsvFunctionsSuite extends QueryTest with SharedSparkSession {
exception = intercept[AnalysisException] {
df.select(from_csv($"value", schema, Map("mode" -> "DROPMALFORMED"))).collect()
},
condition = "_LEGACY_ERROR_TEMP_1099",
condition = "PARSE_MODE_UNSUPPORTED",
parameters = Map(
"funcName" -> "from_csv",
"mode" -> "DROPMALFORMED",
Expand Down
2 changes: 1 addition & 1 deletion sql/core/src/test/scala/org/apache/spark/sql/JsonFunctionsSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -861,7 +861,7 @@ class JsonFunctionsSuite extends QueryTest with SharedSparkSession {
exception = intercept[AnalysisException] {
df.select(from_json($"value", schema, Map("mode" -> "DROPMALFORMED"))).collect()
},
condition = "_LEGACY_ERROR_TEMP_1099",
condition = "PARSE_MODE_UNSUPPORTED",
parameters = Map(
"funcName" -> "from_json",
"mode" -> "DROPMALFORMED",
Expand Down
2 changes: 1 addition & 1 deletion sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/xml/XmlSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1315,7 +1315,7 @@ class XmlSuite
spark.sql(s"""SELECT schema_of_xml('<ROW><a>1<ROW>', map('mode', 'DROPMALFORMED'))""")
.collect()
},
condition = "_LEGACY_ERROR_TEMP_1099",
condition = "PARSE_MODE_UNSUPPORTED",
parameters = Map(
"funcName" -> "schema_of_xml",
"mode" -> "DROPMALFORMED",
Expand Down