Skip to content

Commit c3584dc

Browse files
itholicMaxGekk
itholic
authored and
MaxGekk
committed
[SPARK-42314][SQL] Assign name to _LEGACY_ERROR_TEMP_2127
### What changes were proposed in this pull request? This PR proposes to assign name to _LEGACY_ERROR_TEMP_2127, "DUPLICATED_MAP_KEY". ### Why are the changes needed? We should assign proper name to _LEGACY_ERROR_TEMP_* ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? `./build/sbt "sql/testOnly org.apache.spark.sql.SQLQueryTestSuite*` Closes #39890 from itholic/LEGACY_2127. Lead-authored-by: itholic <haejoon.lee@databricks.com> Co-authored-by: Haejoon Lee <44108233+itholic@users.noreply.github.com> Signed-off-by: Max Gekk <max.gekk@gmail.com> (cherry picked from commit 4b50a46) Signed-off-by: Max Gekk <max.gekk@gmail.com>
1 parent a2dbde3 commit c3584dc

File tree

6 files changed

+86
-31
lines changed

6 files changed

+86
-31
lines changed

‎core/src/main/resources/error/error-classes.json‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -506,6 +506,12 @@
506506
],
507507
"sqlState" : "22012"
508508
},
509+
"DUPLICATED_MAP_KEY" : {
510+
"message" : [
511+
"Duplicate map key <key> was found, please check the input data. If you want to remove the duplicated keys, you can set <mapKeyDedupPolicy> to \"LAST_WIN\" so that the key inserted at last takes precedence."
512+
],
513+
"sqlState" : "23505"
514+
},
509515
"DUPLICATE_KEY" : {
510516
"message" : [
511517
"Found duplicate keys <keyColumn>."
@@ -4197,11 +4203,6 @@
41974203
"Unsuccessful attempt to build maps with <size> elements due to exceeding the map size limit <maxRoundedArrayLength>."
41984204
]
41994205
},
4200-
"_LEGACY_ERROR_TEMP_2127" : {
4201-
"message" : [
4202-
"Duplicate map key <key> was found, please check the input data. If you want to remove the duplicated keys, you can set <mapKeyDedupPolicy> to <lastWin> so that the key inserted at last takes precedence."
4203-
]
4204-
},
42054206
"_LEGACY_ERROR_TEMP_2128" : {
42064207
"message" : [
42074208
"The key array and value array of MapData must have the same length."

‎sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1330,11 +1330,10 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase {
13301330

13311331
def duplicateMapKeyFoundError(key: Any): SparkRuntimeException = {
13321332
new SparkRuntimeException(
1333-
errorClass = "_LEGACY_ERROR_TEMP_2127",
1333+
errorClass = "DUPLICATED_MAP_KEY",
13341334
messageParameters = Map(
13351335
"key" -> key.toString(),
1336-
"mapKeyDedupPolicy" -> toSQLConf(SQLConf.MAP_KEY_DEDUP_POLICY.key),
1337-
"lastWin" -> toSQLConf(SQLConf.MapKeyDedupPolicy.LAST_WIN.toString())))
1336+
"mapKeyDedupPolicy" -> toSQLConf(SQLConf.MAP_KEY_DEDUP_POLICY.key)))
13381337
}
13391338

13401339
def mapDataKeyArrayLengthDiffersFromValueArrayLengthError(): SparkRuntimeException = {

‎sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala‎

Lines changed: 21 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -169,8 +169,13 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
169169
MapType(IntegerType, IntegerType, valueContainsNull = true))
170170
val mNull = Literal.create(null, MapType(StringType, StringType))
171171

172-
checkExceptionInExpression[RuntimeException](
173-
MapConcat(Seq(m0, m1)), "Duplicate map key")
172+
checkErrorInExpression[SparkRuntimeException](
173+
MapConcat(Seq(m0, m1)),
174+
errorClass = "DUPLICATED_MAP_KEY",
175+
parameters = Map(
176+
"key" -> "a",
177+
"mapKeyDedupPolicy" -> "\"spark.sql.mapKeyDedupPolicy\"")
178+
)
174179
withSQLConf(SQLConf.MAP_KEY_DEDUP_POLICY.key -> SQLConf.MapKeyDedupPolicy.LAST_WIN.toString) {
175180
// overlapping maps should remove duplicated map keys w.r.t. last win policy.
176181
checkEvaluation(MapConcat(Seq(m0, m1)), create_map("a" -> "4", "b" -> "2", "c" -> "3"))
@@ -324,8 +329,13 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
324329
checkEvaluation(MapFromEntries(ai2), Map.empty)
325330
checkEvaluation(MapFromEntries(ai3), null)
326331

327-
checkExceptionInExpression[RuntimeException](
328-
MapFromEntries(ai4), "Duplicate map key")
332+
checkErrorInExpression[SparkRuntimeException](
333+
MapFromEntries(ai4),
334+
errorClass = "DUPLICATED_MAP_KEY",
335+
parameters = Map(
336+
"key" -> "1",
337+
"mapKeyDedupPolicy" -> "\"spark.sql.mapKeyDedupPolicy\"")
338+
)
329339
withSQLConf(SQLConf.MAP_KEY_DEDUP_POLICY.key -> SQLConf.MapKeyDedupPolicy.LAST_WIN.toString) {
330340
// Duplicated map keys will be removed w.r.t. the last wins policy.
331341
checkEvaluation(MapFromEntries(ai4), create_map(1 -> 20))
@@ -351,8 +361,13 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
351361
checkEvaluation(MapFromEntries(as2), Map.empty)
352362
checkEvaluation(MapFromEntries(as3), null)
353363

354-
checkExceptionInExpression[RuntimeException](
355-
MapFromEntries(as4), "Duplicate map key")
364+
checkErrorInExpression[SparkRuntimeException](
365+
MapFromEntries(as4),
366+
errorClass = "DUPLICATED_MAP_KEY",
367+
parameters = Map(
368+
"key" -> "a",
369+
"mapKeyDedupPolicy" -> "\"spark.sql.mapKeyDedupPolicy\"")
370+
)
356371
withSQLConf(SQLConf.MAP_KEY_DEDUP_POLICY.key -> SQLConf.MapKeyDedupPolicy.LAST_WIN.toString) {
357372
// Duplicated map keys will be removed w.r.t. the last wins policy.
358373
checkEvaluation(MapFromEntries(as4), create_map("a" -> "bb"))

‎sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala‎

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -282,8 +282,13 @@ class ComplexTypeSuite extends SparkFunSuite with ExpressionEvalHelper {
282282
CreateMap(interlace(strWithNull, intSeq.map(Literal(_)))),
283283
"NULL_MAP_KEY")
284284

285-
checkExceptionInExpression[RuntimeException](
286-
CreateMap(Seq(Literal(1), Literal(2), Literal(1), Literal(3))), "Duplicate map key")
285+
checkErrorInExpression[SparkRuntimeException](
286+
CreateMap(Seq(Literal(1), Literal(2), Literal(1), Literal(3))),
287+
errorClass = "DUPLICATED_MAP_KEY",
288+
parameters = Map(
289+
"key" -> "1",
290+
"mapKeyDedupPolicy" -> "\"spark.sql.mapKeyDedupPolicy\"")
291+
)
287292
withSQLConf(SQLConf.MAP_KEY_DEDUP_POLICY.key -> SQLConf.MapKeyDedupPolicy.LAST_WIN.toString) {
288293
// Duplicated map keys will be removed w.r.t. the last wins policy.
289294
checkEvaluation(
@@ -388,11 +393,15 @@ class ComplexTypeSuite extends SparkFunSuite with ExpressionEvalHelper {
388393
MapFromArrays(intWithNullArray, strArray),
389394
"NULL_MAP_KEY")
390395

391-
checkExceptionInExpression[RuntimeException](
396+
checkErrorInExpression[SparkRuntimeException](
392397
MapFromArrays(
393398
Literal.create(Seq(1, 1), ArrayType(IntegerType)),
394399
Literal.create(Seq(2, 3), ArrayType(IntegerType))),
395-
"Duplicate map key")
400+
errorClass = "DUPLICATED_MAP_KEY",
401+
parameters = Map(
402+
"key" -> "1",
403+
"mapKeyDedupPolicy" -> "\"spark.sql.mapKeyDedupPolicy\"")
404+
)
396405
withSQLConf(SQLConf.MAP_KEY_DEDUP_POLICY.key -> SQLConf.MapKeyDedupPolicy.LAST_WIN.toString) {
397406
// Duplicated map keys will be removed w.r.t. the last wins policy.
398407
checkEvaluation(
@@ -512,8 +521,13 @@ class ComplexTypeSuite extends SparkFunSuite with ExpressionEvalHelper {
512521
val m6 = Map("a" -> "1", "b" -> "2", "c" -> "3")
513522
checkEvaluation(StringToMap(s6, NonFoldableLiteral("&"), NonFoldableLiteral("=")), m6)
514523

515-
checkExceptionInExpression[RuntimeException](
516-
new StringToMap(Literal("a:1,b:2,a:3")), "Duplicate map key")
524+
checkErrorInExpression[SparkRuntimeException](
525+
new StringToMap(Literal("a:1,b:2,a:3")),
526+
errorClass = "DUPLICATED_MAP_KEY",
527+
parameters = Map(
528+
"key" -> "a",
529+
"mapKeyDedupPolicy" -> "\"spark.sql.mapKeyDedupPolicy\"")
530+
)
517531
withSQLConf(SQLConf.MAP_KEY_DEDUP_POLICY.key -> SQLConf.MapKeyDedupPolicy.LAST_WIN.toString) {
518532
// Duplicated map keys will be removed w.r.t. the last wins policy.
519533
checkEvaluation(

‎sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/HigherOrderFunctionsSuite.scala‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717

1818
package org.apache.spark.sql.catalyst.expressions
1919

20-
import org.apache.spark.{SparkException, SparkFunSuite}
20+
import org.apache.spark.{SparkException, SparkFunSuite, SparkRuntimeException}
2121
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
2222
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.DataTypeMismatch
2323
import org.apache.spark.sql.catalyst.expressions.Cast._
@@ -469,8 +469,13 @@ class HigherOrderFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper
469469
transformKeys(transformKeys(ai0, plusOne), plusValue),
470470
create_map(3 -> 1, 5 -> 2, 7 -> 3, 9 -> 4))
471471

472-
checkExceptionInExpression[RuntimeException](
473-
transformKeys(ai0, modKey), "Duplicate map key")
472+
checkErrorInExpression[SparkRuntimeException](
473+
transformKeys(ai0, modKey),
474+
errorClass = "DUPLICATED_MAP_KEY",
475+
parameters = Map(
476+
"key" -> "1",
477+
"mapKeyDedupPolicy" -> "\"spark.sql.mapKeyDedupPolicy\"")
478+
)
474479
withSQLConf(SQLConf.MAP_KEY_DEDUP_POLICY.key -> SQLConf.MapKeyDedupPolicy.LAST_WIN.toString) {
475480
// Duplicated map keys will be removed w.r.t. the last wins policy.
476481
checkEvaluation(transformKeys(ai0, modKey), create_map(1 -> 4, 2 -> 2, 0 -> 3))

‎sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/ArrayBasedMapBuilderSuite.scala‎

Lines changed: 29 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -51,8 +51,13 @@ class ArrayBasedMapBuilderSuite extends SparkFunSuite with SQLHelper {
5151
test("fail while duplicated keys detected") {
5252
val builder = new ArrayBasedMapBuilder(IntegerType, IntegerType)
5353
builder.put(1, 1)
54-
val e = intercept[RuntimeException](builder.put(1, 2))
55-
assert(e.getMessage.contains("Duplicate map key 1 was found"))
54+
checkError(
55+
exception = intercept[SparkRuntimeException](builder.put(1, 2)),
56+
errorClass = "DUPLICATED_MAP_KEY",
57+
parameters = Map(
58+
"key" -> "1",
59+
"mapKeyDedupPolicy" -> "\"spark.sql.mapKeyDedupPolicy\"")
60+
)
5661
}
5762

5863
test("remove duplicated keys with last wins policy") {
@@ -71,9 +76,15 @@ class ArrayBasedMapBuilderSuite extends SparkFunSuite with SQLHelper {
7176
val builder = new ArrayBasedMapBuilder(BinaryType, IntegerType)
7277
builder.put(Array(1.toByte), 1)
7378
builder.put(Array(2.toByte), 2)
74-
val e = intercept[RuntimeException](builder.put(Array(1.toByte), 3))
7579
// By default duplicated map key fails the query.
76-
assert(e.getMessage.contains("Duplicate map key"))
80+
val arr = Array(1.toByte)
81+
checkError(
82+
exception = intercept[SparkRuntimeException](builder.put(arr, 3)),
83+
errorClass = "DUPLICATED_MAP_KEY",
84+
parameters = Map(
85+
"key" -> arr.toString,
86+
"mapKeyDedupPolicy" -> "\"spark.sql.mapKeyDedupPolicy\"")
87+
)
7788

7889
withSQLConf(SQLConf.MAP_KEY_DEDUP_POLICY.key -> SQLConf.MapKeyDedupPolicy.LAST_WIN.toString) {
7990
val builder = new ArrayBasedMapBuilder(BinaryType, IntegerType)
@@ -102,9 +113,14 @@ class ArrayBasedMapBuilderSuite extends SparkFunSuite with SQLHelper {
102113
val builder = new ArrayBasedMapBuilder(new StructType().add("i", "int"), IntegerType)
103114
builder.put(InternalRow(1), 1)
104115
builder.put(InternalRow(2), 2)
105-
val e = intercept[RuntimeException](builder.put(unsafeRow, 3))
106116
// By default duplicated map key fails the query.
107-
assert(e.getMessage.contains("Duplicate map key"))
117+
checkError(
118+
exception = intercept[SparkRuntimeException](builder.put(unsafeRow, 3)),
119+
errorClass = "DUPLICATED_MAP_KEY",
120+
parameters = Map(
121+
"key" -> "[0,1]",
122+
"mapKeyDedupPolicy" -> "\"spark.sql.mapKeyDedupPolicy\"")
123+
)
108124

109125
withSQLConf(SQLConf.MAP_KEY_DEDUP_POLICY.key -> SQLConf.MapKeyDedupPolicy.LAST_WIN.toString) {
110126
val builder = new ArrayBasedMapBuilder(new StructType().add("i", "int"), IntegerType)
@@ -131,9 +147,14 @@ class ArrayBasedMapBuilderSuite extends SparkFunSuite with SQLHelper {
131147
val builder = new ArrayBasedMapBuilder(ArrayType(IntegerType), IntegerType)
132148
builder.put(new GenericArrayData(Seq(1, 1)), 1)
133149
builder.put(new GenericArrayData(Seq(2, 2)), 2)
134-
val e = intercept[RuntimeException](builder.put(unsafeArray, 3))
135150
// By default duplicated map key fails the query.
136-
assert(e.getMessage.contains("Duplicate map key"))
151+
checkError(
152+
exception = intercept[SparkRuntimeException](builder.put(unsafeArray, 3)),
153+
errorClass = "DUPLICATED_MAP_KEY",
154+
parameters = Map(
155+
"key" -> unsafeArray.toString,
156+
"mapKeyDedupPolicy" -> "\"spark.sql.mapKeyDedupPolicy\"")
157+
)
137158

138159
withSQLConf(SQLConf.MAP_KEY_DEDUP_POLICY.key -> SQLConf.MapKeyDedupPolicy.LAST_WIN.toString) {
139160
val builder = new ArrayBasedMapBuilder(ArrayType(IntegerType), IntegerType)

0 commit comments

Comments
 (0)