Skip to content

Commit 2b7ab81

Browse files
willbmarmbrus
authored andcommitted
[SPARK-3329][SQL] Don't depend on Hive SET pair ordering in tests.
This fixes some possible spurious test failures in `HiveQuerySuite` by comparing sets of key-value pairs as sets, rather than as lists. Author: William Benton <willb@redhat.com> Author: Aaron Davidson <aaron@databricks.com> Closes #2220 from willb/spark-3329 and squashes the following commits: 3b3e205 [William Benton] Collapse collectResults case match in HiveQuerySuite 6525d8e [William Benton] Handle cases where SET returns Rows of (single) strings cf11b0e [Aaron Davidson] Fix flakey HiveQuerySuite test
1 parent dc1dbf2 commit 2b7ab81

File tree

1 file changed

+26
-21
lines changed

1 file changed

+26
-21
lines changed

sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala

Lines changed: 26 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -558,62 +558,67 @@ class HiveQuerySuite extends HiveComparisonTest {
558558
val testKey = "spark.sql.key.usedfortestonly"
559559
val testVal = "test.val.0"
560560
val nonexistentKey = "nonexistent"
561-
561+
val KV = "([^=]+)=([^=]*)".r
562+
def collectResults(rdd: SchemaRDD): Set[(String, String)] =
563+
rdd.collect().map {
564+
case Row(key: String, value: String) => key -> value
565+
case Row(KV(key, value)) => key -> value
566+
}.toSet
562567
clear()
563568

564569
// "set" itself returns all config variables currently specified in SQLConf.
565570
// TODO: Should we be listing the default here always? probably...
566571
assert(sql("SET").collect().size == 0)
567572

568-
assertResult(Array(s"$testKey=$testVal")) {
569-
sql(s"SET $testKey=$testVal").collect().map(_.getString(0))
573+
assertResult(Set(testKey -> testVal)) {
574+
collectResults(hql(s"SET $testKey=$testVal"))
570575
}
571576

572577
assert(hiveconf.get(testKey, "") == testVal)
573-
assertResult(Array(s"$testKey=$testVal")) {
574-
sql(s"SET $testKey=$testVal").collect().map(_.getString(0))
578+
assertResult(Set(testKey -> testVal)) {
579+
collectResults(hql("SET"))
575580
}
576581

577582
sql(s"SET ${testKey + testKey}=${testVal + testVal}")
578583
assert(hiveconf.get(testKey + testKey, "") == testVal + testVal)
579-
assertResult(Array(s"$testKey=$testVal", s"${testKey + testKey}=${testVal + testVal}")) {
580-
sql(s"SET").collect().map(_.getString(0))
584+
assertResult(Set(testKey -> testVal, (testKey + testKey) -> (testVal + testVal))) {
585+
collectResults(hql("SET"))
581586
}
582587

583588
// "set key"
584-
assertResult(Array(s"$testKey=$testVal")) {
585-
sql(s"SET $testKey").collect().map(_.getString(0))
589+
assertResult(Set(testKey -> testVal)) {
590+
collectResults(hql(s"SET $testKey"))
586591
}
587592

588-
assertResult(Array(s"$nonexistentKey=<undefined>")) {
589-
sql(s"SET $nonexistentKey").collect().map(_.getString(0))
593+
assertResult(Set(nonexistentKey -> "<undefined>")) {
594+
collectResults(hql(s"SET $nonexistentKey"))
590595
}
591596

592597
// Assert that sql() should have the same effects as sql() by repeating the above using sql().
593598
clear()
594599
assert(sql("SET").collect().size == 0)
595600

596-
assertResult(Array(s"$testKey=$testVal")) {
597-
sql(s"SET $testKey=$testVal").collect().map(_.getString(0))
601+
assertResult(Set(testKey -> testVal)) {
602+
collectResults(sql(s"SET $testKey=$testVal"))
598603
}
599604

600605
assert(hiveconf.get(testKey, "") == testVal)
601-
assertResult(Array(s"$testKey=$testVal")) {
602-
sql("SET").collect().map(_.getString(0))
606+
assertResult(Set(testKey -> testVal)) {
607+
collectResults(sql("SET"))
603608
}
604609

605610
sql(s"SET ${testKey + testKey}=${testVal + testVal}")
606611
assert(hiveconf.get(testKey + testKey, "") == testVal + testVal)
607-
assertResult(Array(s"$testKey=$testVal", s"${testKey + testKey}=${testVal + testVal}")) {
608-
sql("SET").collect().map(_.getString(0))
612+
assertResult(Set(testKey -> testVal, (testKey + testKey) -> (testVal + testVal))) {
613+
collectResults(sql("SET"))
609614
}
610615

611-
assertResult(Array(s"$testKey=$testVal")) {
612-
sql(s"SET $testKey").collect().map(_.getString(0))
616+
assertResult(Set(testKey -> testVal)) {
617+
collectResults(sql(s"SET $testKey"))
613618
}
614619

615-
assertResult(Array(s"$nonexistentKey=<undefined>")) {
616-
sql(s"SET $nonexistentKey").collect().map(_.getString(0))
620+
assertResult(Set(nonexistentKey -> "<undefined>")) {
621+
collectResults(sql(s"SET $nonexistentKey"))
617622
}
618623

619624
clear()

0 commit comments

Comments
 (0)