Skip to content

Commit ea3b1e3

Browse files
10110346gatorsmile
authored andcommitted
[SPARK-20763][SQL] The function of month and day return the value which is not we expected.
## What changes were proposed in this pull request? spark-sql>select month("1582-09-28"); spark-sql>10 For this case, the expected result is 9, but it is 10. spark-sql>select day("1582-04-18"); spark-sql>28 For this case, the expected result is 18, but it is 28. when the date before "1582-10-04", the function of `month` and `day` return the value which is not we expected. ## How was this patch tested? unit tests Author: liuxian <liu.xian3@zte.com.cn> Closes #17997 from 10110346/wip_lx_0516.
1 parent bff021d commit ea3b1e3

File tree

2 files changed

+26
-1
lines changed

2 files changed

+26
-1
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -603,7 +603,14 @@ object DateTimeUtils {
603603
*/
604604
private[this] def getYearAndDayInYear(daysSince1970: SQLDate): (Int, Int) = {
605605
// add the difference (in days) between 1.1.1970 and the artificial year 0 (-17999)
606-
val daysNormalized = daysSince1970 + toYearZero
606+
var daysSince1970Tmp = daysSince1970
607+
// Since Julian calendar was replaced with the Gregorian calendar,
608+
// the 10 days after Oct. 4 were skipped.
609+
// (1582-10-04) -141428 days since 1970-01-01
610+
if (daysSince1970 <= -141428) {
611+
daysSince1970Tmp -= 10
612+
}
613+
val daysNormalized = daysSince1970Tmp + toYearZero
607614
val numOfQuarterCenturies = daysNormalized / daysIn400Years
608615
val daysInThis400 = daysNormalized % daysIn400Years + 1
609616
val (years, dayInYear) = numYears(daysInThis400)

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,9 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
7676
}
7777
}
7878
checkEvaluation(DayOfYear(Literal.create(null, DateType)), null)
79+
80+
checkEvaluation(DayOfYear(Literal(new Date(sdf.parse("1582-10-15 13:10:15").getTime))), 288)
81+
checkEvaluation(DayOfYear(Literal(new Date(sdf.parse("1582-10-04 13:10:15").getTime))), 277)
7982
checkConsistencyBetweenInterpretedAndCodegen(DayOfYear, DateType)
8083
}
8184

@@ -96,6 +99,8 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
9699
}
97100
}
98101
}
102+
checkEvaluation(Year(Literal(new Date(sdf.parse("1582-01-01 13:10:15").getTime))), 1582)
103+
checkEvaluation(Year(Literal(new Date(sdf.parse("1581-12-31 13:10:15").getTime))), 1581)
99104
checkConsistencyBetweenInterpretedAndCodegen(Year, DateType)
100105
}
101106

@@ -116,6 +121,9 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
116121
}
117122
}
118123
}
124+
125+
checkEvaluation(Quarter(Literal(new Date(sdf.parse("1582-10-01 13:10:15").getTime))), 4)
126+
checkEvaluation(Quarter(Literal(new Date(sdf.parse("1582-09-30 13:10:15").getTime))), 3)
119127
checkConsistencyBetweenInterpretedAndCodegen(Quarter, DateType)
120128
}
121129

@@ -125,6 +133,10 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
125133
checkEvaluation(Month(Cast(Literal(sdfDate.format(d)), DateType, gmtId)), 4)
126134
checkEvaluation(Month(Cast(Literal(ts), DateType, gmtId)), 11)
127135

136+
checkEvaluation(Month(Literal(new Date(sdf.parse("1582-04-28 13:10:15").getTime))), 4)
137+
checkEvaluation(Month(Literal(new Date(sdf.parse("1582-10-04 13:10:15").getTime))), 10)
138+
checkEvaluation(Month(Literal(new Date(sdf.parse("1582-10-15 13:10:15").getTime))), 10)
139+
128140
val c = Calendar.getInstance()
129141
(2003 to 2004).foreach { y =>
130142
(0 to 3).foreach { m =>
@@ -146,6 +158,10 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
146158
checkEvaluation(DayOfMonth(Cast(Literal(sdfDate.format(d)), DateType, gmtId)), 8)
147159
checkEvaluation(DayOfMonth(Cast(Literal(ts), DateType, gmtId)), 8)
148160

161+
checkEvaluation(DayOfMonth(Literal(new Date(sdf.parse("1582-04-28 13:10:15").getTime))), 28)
162+
checkEvaluation(DayOfMonth(Literal(new Date(sdf.parse("1582-10-15 13:10:15").getTime))), 15)
163+
checkEvaluation(DayOfMonth(Literal(new Date(sdf.parse("1582-10-04 13:10:15").getTime))), 4)
164+
149165
val c = Calendar.getInstance()
150166
(1999 to 2000).foreach { y =>
151167
c.set(y, 0, 1, 0, 0, 0)
@@ -186,6 +202,8 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
186202
checkEvaluation(WeekOfYear(Cast(Literal(sdfDate.format(d)), DateType, gmtId)), 15)
187203
checkEvaluation(WeekOfYear(Cast(Literal(ts), DateType, gmtId)), 45)
188204
checkEvaluation(WeekOfYear(Cast(Literal("2011-05-06"), DateType, gmtId)), 18)
205+
checkEvaluation(WeekOfYear(Literal(new Date(sdf.parse("1582-10-15 13:10:15").getTime))), 40)
206+
checkEvaluation(WeekOfYear(Literal(new Date(sdf.parse("1582-10-04 13:10:15").getTime))), 40)
189207
checkConsistencyBetweenInterpretedAndCodegen(WeekOfYear, DateType)
190208
}
191209

0 commit comments

Comments
 (0)