Skip to content

Commit

Permalink
[SPARK-46524][SQL] Improve error messages for invalid save mode
Browse files Browse the repository at this point in the history
### What changes were proposed in this pull request?

This PR improves the error messages when writing a data frame with an invalid save mode.

### Why are the changes needed?

To improve the error messages.
Before this PR, Spark throws an java.lang.IllegalArgumentException:
`java.lang.IllegalArgumentException: Unknown save mode: foo. Accepted save modes are 'overwrite', 'append', 'ignore', 'error', 'errorifexists', 'default'.`

After this PR, the error will have a proper error class:
`[INVALID_SAVE_MODE] The specified save mode "foo" is invalid. Valid save modes include "append", "overwrite", "ignore", "error", "errorifexists", and "default"."
`

### Does this PR introduce _any_ user-facing change?

Yes. The error messages will be changed.

### How was this patch tested?

New unit test

### Was this patch authored or co-authored using generative AI tooling?

No

Closes apache#44508 from allisonwang-db/spark-46524-invalid-save-mode.

Authored-by: allisonwang-db <allison.wang@databricks.com>
Signed-off-by: Max Gekk <max.gekk@gmail.com>
  • Loading branch information
allisonwang-db authored and MaxGekk committed Jan 3, 2024
1 parent f844e78 commit a3d9992
Show file tree
Hide file tree
Showing 6 changed files with 30 additions and 3 deletions.
2 changes: 1 addition & 1 deletion R/pkg/tests/fulltests/test_sparkSQL.R
Original file line number Diff line number Diff line change
Expand Up @@ -1414,7 +1414,7 @@ test_that("test HiveContext", {

# Invalid mode
expect_error(saveAsTable(df, "parquetest", "parquet", mode = "abc", path = parquetDataPath),
"illegal argument - Unknown save mode: abc")
"Error in mode : analysis error - \\[INVALID_SAVE_MODE\\].*")
unsetHiveContext()
}
})
Expand Down
6 changes: 6 additions & 0 deletions common/utils/src/main/resources/error/error-classes.json
Original file line number Diff line number Diff line change
Expand Up @@ -2239,6 +2239,12 @@
],
"sqlState" : "42613"
},
"INVALID_SAVE_MODE" : {
"message" : [
"The specified save mode <mode> is invalid. Valid save modes include \"append\", \"overwrite\", \"ignore\", \"error\", \"errorifexists\", and \"default\"."
],
"sqlState" : "42000"
},
"INVALID_SCHEMA" : {
"message" : [
"The input schema <inputSchema> is not a valid schema string."
Expand Down
6 changes: 6 additions & 0 deletions docs/sql-error-conditions.md
Original file line number Diff line number Diff line change
Expand Up @@ -1271,6 +1271,12 @@ For more details see [INVALID_PARTITION_OPERATION](sql-error-conditions-invalid-

Parameterized query must either use positional, or named parameters, but not both.

### INVALID_SAVE_MODE

[SQLSTATE: 42000](sql-error-conditions-sqlstates.html#class-42-syntax-error-or-access-rule-violation)

The specified save mode `<mode>` is invalid. Valid save modes include "append", "overwrite", "ignore", "error", "errorifexists", and "default".

### [INVALID_SCHEMA](sql-error-conditions-invalid-schema-error-class.html)

[SQLSTATE: 42K07](sql-error-conditions-sqlstates.html#class-42-syntax-error-or-access-rule-violation)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3184,6 +3184,13 @@ private[sql] object QueryCompilationErrors extends QueryErrorsBase with Compilat
"config" -> SQLConf.LEGACY_PATH_OPTION_BEHAVIOR.key))
}

def invalidSaveModeError(saveMode: String): Throwable = {
new AnalysisException(
errorClass = "INVALID_SAVE_MODE",
messageParameters = Map("mode" -> toDSOption(saveMode))
)
}

def writeWithSaveModeUnsupportedBySourceError(source: String, createMode: String): Throwable = {
new AnalysisException(
errorClass = "_LEGACY_ERROR_TEMP_1308",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -88,8 +88,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
case "append" => mode(SaveMode.Append)
case "ignore" => mode(SaveMode.Ignore)
case "error" | "errorifexists" | "default" => mode(SaveMode.ErrorIfExists)
case _ => throw new IllegalArgumentException(s"Unknown save mode: $saveMode. Accepted " +
"save modes are 'overwrite', 'append', 'ignore', 'error', 'errorifexists', 'default'.")
case _ => throw QueryCompilationErrors.invalidSaveModeError(saveMode)
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -629,6 +629,15 @@ class PythonDataSourceSuite extends QueryTest with SharedSparkSession {
assert(error.getMessage.contains("TableProvider implementation SimpleDataSource " +
"cannot be written with ErrorIfExists mode, please use Append or Overwrite modes instead."))
}

withClue("invalid mode") {
checkError(
exception = intercept[AnalysisException] {
spark.range(1).write.format(dataSourceName).mode("foo").save()
},
errorClass = "INVALID_SAVE_MODE",
parameters = Map("mode" -> "\"foo\""))
}
}

test("data source write - overwrite mode") {
Expand Down

0 comments on commit a3d9992

Please sign in to comment.