Skip to content

Commit 252b6cf

Browse files
MaxGekkdongjoon-hyun
authored andcommitted
[SPARK-29187][SQL] Return null from date_part() for the null field
### What changes were proposed in this pull request? In the PR, I propose to change behavior of the `date_part()` function in handling `null` field, and make it the same as PostgreSQL has. If `field` parameter is `null`, the function should return `null` of the `double` type as PostgreSQL does: ```sql # select date_part(null, date '2019-09-20'); date_part ----------- (1 row) # select pg_typeof(date_part(null, date '2019-09-20')); pg_typeof ------------------ double precision (1 row) ``` ### Why are the changes needed? The `date_part()` function was added to maintain feature parity with PostgreSQL but current behavior of the function is different in handling null as `field`. ### Does this PR introduce any user-facing change? Yes. Before: ```sql spark-sql> select date_part(null, date'2019-09-20'); Error in query: null; line 1 pos 7 ``` After: ```sql spark-sql> select date_part(null, date'2019-09-20'); NULL ``` ### How was this patch tested? Add new tests to `DateFunctionsSuite for 2 cases: - `field` = `null`, `source` = a date literal - `field` = `null`, `source` = a date column Closes #25865 from MaxGekk/date_part-null. Authored-by: Maxim Gekk <max.gekk@gmail.com> Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
1 parent ff3a737 commit 252b6cf

File tree

4 files changed

+30
-5
lines changed

4 files changed

+30
-5
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2053,10 +2053,15 @@ case class DatePart(field: Expression, source: Expression, child: Expression)
20532053
if (!field.foldable) {
20542054
throw new AnalysisException("The field parameter needs to be a foldable string value.")
20552055
}
2056-
val fieldStr = field.eval().asInstanceOf[UTF8String].toString
2057-
DatePart.parseExtractField(fieldStr, source, {
2058-
throw new AnalysisException(s"Literals of type '$fieldStr' are currently not supported.")
2059-
})
2056+
val fieldEval = field.eval()
2057+
if (fieldEval == null) {
2058+
Literal(null, DoubleType)
2059+
} else {
2060+
val fieldStr = fieldEval.asInstanceOf[UTF8String].toString
2061+
DatePart.parseExtractField(fieldStr, source, {
2062+
throw new AnalysisException(s"Literals of type '$fieldStr' are currently not supported.")
2063+
})
2064+
}
20602065
})
20612066
}
20622067

sql/core/src/test/resources/sql-tests/inputs/date_part.sql

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,3 +66,5 @@ select date_part('secs', c) from t;
6666
select date_part('not_supported', c) from t;
6767

6868
select date_part(c, c) from t;
69+
70+
select date_part(null, c) from t;

sql/core/src/test/resources/sql-tests/results/date_part.sql.out

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
-- Automatically generated by SQLQueryTestSuite
2-
-- Number of queries: 51
2+
-- Number of queries: 52
33

44

55
-- !query 0
@@ -410,3 +410,11 @@ struct<>
410410
-- !query 50 output
411411
org.apache.spark.sql.AnalysisException
412412
The field parameter needs to be a foldable string value.;; line 1 pos 7
413+
414+
415+
-- !query 51
416+
select date_part(null, c) from t
417+
-- !query 51 schema
418+
struct<date_part(NULL, t.`c`):double>
419+
-- !query 51 output
420+
NULL

sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import org.apache.spark.sql.catalyst.util.DateTimeUtils
2727
import org.apache.spark.sql.functions._
2828
import org.apache.spark.sql.internal.SQLConf
2929
import org.apache.spark.sql.test.SharedSparkSession
30+
import org.apache.spark.sql.types.{DoubleType, StructField, StructType}
3031
import org.apache.spark.unsafe.types.CalendarInterval
3132

3233
class DateFunctionsSuite extends QueryTest with SharedSparkSession {
@@ -796,4 +797,13 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession {
796797
Seq(Row(Instant.parse(timestamp))))
797798
}
798799
}
800+
801+
test("handling null field by date_part") {
802+
val input = Seq(Date.valueOf("2019-09-20")).toDF("d")
803+
Seq("date_part(null, d)", "date_part(null, date'2019-09-20')").foreach { expr =>
804+
val df = input.selectExpr(expr)
805+
assert(df.schema.headOption.get.dataType == DoubleType)
806+
checkAnswer(df, Row(null))
807+
}
808+
}
799809
}

0 commit comments

Comments
 (0)