Skip to content

Commit f09a9e9

Browse files
ueshingatorsmile
authored andcommitted
[SPARK-24007][SQL] EqualNullSafe for FloatType and DoubleType might generate a wrong result by codegen.
## What changes were proposed in this pull request? `EqualNullSafe` for `FloatType` and `DoubleType` might generate a wrong result by codegen. ```scala scala> val df = Seq((Some(-1.0d), None), (None, Some(-1.0d))).toDF() df: org.apache.spark.sql.DataFrame = [_1: double, _2: double] scala> df.show() +----+----+ | _1| _2| +----+----+ |-1.0|null| |null|-1.0| +----+----+ scala> df.filter("_1 <=> _2").show() +----+----+ | _1| _2| +----+----+ |-1.0|null| |null|-1.0| +----+----+ ``` The result should be empty but the result remains two rows. ## How was this patch tested? Added a test. Author: Takuya UESHIN <ueshin@databricks.com> Closes #21094 from ueshin/issues/SPARK-24007/equalnullsafe.
1 parent f81fa47 commit f09a9e9

File tree

2 files changed

+11
-2
lines changed

2 files changed

+11
-2
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -582,8 +582,10 @@ class CodegenContext {
582582
*/
583583
def genEqual(dataType: DataType, c1: String, c2: String): String = dataType match {
584584
case BinaryType => s"java.util.Arrays.equals($c1, $c2)"
585-
case FloatType => s"(java.lang.Float.isNaN($c1) && java.lang.Float.isNaN($c2)) || $c1 == $c2"
586-
case DoubleType => s"(java.lang.Double.isNaN($c1) && java.lang.Double.isNaN($c2)) || $c1 == $c2"
585+
case FloatType =>
586+
s"((java.lang.Float.isNaN($c1) && java.lang.Float.isNaN($c2)) || $c1 == $c2)"
587+
case DoubleType =>
588+
s"((java.lang.Double.isNaN($c1) && java.lang.Double.isNaN($c2)) || $c1 == $c2)"
587589
case dt: DataType if isPrimitiveType(dt) => s"$c1 == $c2"
588590
case dt: DataType if dt.isInstanceOf[AtomicType] => s"$c1.equals($c2)"
589591
case array: ArrayType => genComp(array, c1, c2) + " == 0"

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/PredicateSuite.scala

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -442,4 +442,11 @@ class PredicateSuite extends SparkFunSuite with ExpressionEvalHelper {
442442
InSet(Literal(1), Set(1, 2, 3, 4)).genCode(ctx)
443443
assert(ctx.inlinedMutableStates.isEmpty)
444444
}
445+
446+
test("SPARK-24007: EqualNullSafe for FloatType and DoubleType might generate a wrong result") {
447+
checkEvaluation(EqualNullSafe(Literal(null, FloatType), Literal(-1.0f)), false)
448+
checkEvaluation(EqualNullSafe(Literal(-1.0f), Literal(null, FloatType)), false)
449+
checkEvaluation(EqualNullSafe(Literal(null, DoubleType), Literal(-1.0d)), false)
450+
checkEvaluation(EqualNullSafe(Literal(-1.0d), Literal(null, DoubleType)), false)
451+
}
445452
}

0 commit comments

Comments
 (0)