Skip to content

Commit cc4c07d

Browse files
MaxGekkMaxGekk
committed
Pass ZoneId to millisToDays() and daysToMillis()
1 parent 9a2b7be commit cc4c07d

File tree

5 files changed

+45
-40
lines changed

5 files changed

+45
-40
lines changed

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,7 @@ case class CurrentBatchTimestamp(
134134
def toLiteral: Literal = dataType match {
135135
case _: TimestampType =>
136136
Literal(DateTimeUtils.fromJavaTimestamp(new Timestamp(timestampMs)), TimestampType)
137-
case _: DateType => Literal(DateTimeUtils.millisToDays(timestampMs, timeZone), DateType)
137+
case _: DateType => Literal(DateTimeUtils.millisToDays(timestampMs, zoneId), DateType)
138138
}
139139
}
140140

@@ -1320,14 +1320,14 @@ case class MonthsBetween(
13201320

13211321
override def nullSafeEval(t1: Any, t2: Any, roundOff: Any): Any = {
13221322
DateTimeUtils.monthsBetween(
1323-
t1.asInstanceOf[Long], t2.asInstanceOf[Long], roundOff.asInstanceOf[Boolean], timeZone)
1323+
t1.asInstanceOf[Long], t2.asInstanceOf[Long], roundOff.asInstanceOf[Boolean], zoneId)
13241324
}
13251325

13261326
override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
1327-
val tz = ctx.addReferenceObj("timeZone", timeZone)
1327+
val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName)
13281328
val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
13291329
defineCodeGen(ctx, ev, (d1, d2, roundOff) => {
1330-
s"""$dtu.monthsBetween($d1, $d2, $roundOff, $tz)"""
1330+
s"""$dtu.monthsBetween($d1, $d2, $roundOff, $zid)"""
13311331
})
13321332
}
13331333

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -67,21 +67,21 @@ object DateTimeUtils {
6767

6868
// we should use the exact day as Int, for example, (year, month, day) -> day
6969
def millisToDays(millisUtc: Long): SQLDate = {
70-
millisToDays(millisUtc, defaultTimeZone())
70+
millisToDays(millisUtc, defaultTimeZone().toZoneId)
7171
}
7272

73-
def millisToDays(millisUtc: Long, timeZone: TimeZone): SQLDate = {
73+
def millisToDays(millisUtc: Long, zoneId: ZoneId): SQLDate = {
7474
val instant = microsToInstant(Math.multiplyExact(millisUtc, MICROS_PER_MILLIS))
75-
localDateToDays(LocalDateTime.ofInstant(instant, timeZone.toZoneId).toLocalDate)
75+
localDateToDays(LocalDateTime.ofInstant(instant, zoneId).toLocalDate)
7676
}
7777

7878
// reverse of millisToDays
7979
def daysToMillis(days: SQLDate): Long = {
80-
daysToMillis(days, defaultTimeZone())
80+
daysToMillis(days, defaultTimeZone().toZoneId)
8181
}
8282

83-
def daysToMillis(days: SQLDate, timeZone: TimeZone): Long = {
84-
val instant = daysToLocalDate(days).atStartOfDay(timeZone.toZoneId).toInstant
83+
def daysToMillis(days: SQLDate, zoneId: ZoneId): Long = {
84+
val instant = daysToLocalDate(days).atStartOfDay(zoneId).toInstant
8585
instantToMicros(instant) / MICROS_PER_MILLIS
8686
}
8787

@@ -585,11 +585,11 @@ object DateTimeUtils {
585585
time1: SQLTimestamp,
586586
time2: SQLTimestamp,
587587
roundOff: Boolean,
588-
timeZone: TimeZone): Double = {
588+
zoneId: ZoneId): Double = {
589589
val millis1 = MICROSECONDS.toMillis(time1)
590590
val millis2 = MICROSECONDS.toMillis(time2)
591-
val date1 = millisToDays(millis1, timeZone)
592-
val date2 = millisToDays(millis2, timeZone)
591+
val date1 = millisToDays(millis1, zoneId)
592+
val date2 = millisToDays(millis2, zoneId)
593593
val (year1, monthInYear1, dayInMonth1, daysToMonthEnd1) = splitDate(date1)
594594
val (year2, monthInYear2, dayInMonth2, daysToMonthEnd2) = splitDate(date2)
595595

@@ -603,8 +603,8 @@ object DateTimeUtils {
603603
}
604604
// using milliseconds can cause precision loss with more than 8 digits
605605
// we follow Hive's implementation which uses seconds
606-
val secondsInDay1 = MILLISECONDS.toSeconds(millis1 - daysToMillis(date1, timeZone))
607-
val secondsInDay2 = MILLISECONDS.toSeconds(millis2 - daysToMillis(date2, timeZone))
606+
val secondsInDay1 = MILLISECONDS.toSeconds(millis1 - daysToMillis(date1, zoneId))
607+
val secondsInDay2 = MILLISECONDS.toSeconds(millis2 - daysToMillis(date2, zoneId))
608608
val secondsDiff = (dayInMonth1 - dayInMonth2) * SECONDS_PER_DAY + secondsInDay1 - secondsInDay2
609609
val secondsInMonth = DAYS.toSeconds(31)
610610
val diff = monthDiff + secondsDiff / secondsInMonth.toDouble
@@ -733,8 +733,8 @@ object DateTimeUtils {
733733
millis += offset
734734
millis - millis % MILLIS_PER_DAY - offset
735735
case _ => // Try to truncate date levels
736-
val dDays = millisToDays(millis, timeZone)
737-
daysToMillis(truncDate(dDays, level), timeZone)
736+
val dDays = millisToDays(millis, timeZone.toZoneId)
737+
daysToMillis(truncDate(dDays, level), timeZone.toZoneId)
738738
}
739739
truncated * MICROS_PER_MILLIS
740740
}

‎sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/UnivocityParserSuite.scala‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ package org.apache.spark.sql.catalyst.csv
1919

2020
import java.math.BigDecimal
2121
import java.text.{DecimalFormat, DecimalFormatSymbols}
22+
import java.time.ZoneOffset
2223
import java.util.{Locale, TimeZone}
2324

2425
import org.apache.commons.lang3.time.FastDateFormat
@@ -137,7 +138,7 @@ class UnivocityParserSuite extends SparkFunSuite with SQLHelper {
137138
val expectedDate = format.parse(customDate).getTime
138139
val castedDate = parser.makeConverter("_1", DateType, nullable = true)
139140
.apply(customDate)
140-
assert(castedDate == DateTimeUtils.millisToDays(expectedDate, TimeZone.getTimeZone("GMT")))
141+
assert(castedDate == DateTimeUtils.millisToDays(expectedDate, ZoneOffset.UTC))
141142

142143
val timestamp = "2015-01-01 00:00:00"
143144
timestampsOptions = new CSVOptions(Map(

‎sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -55,9 +55,9 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
5555
val ts = new Timestamp(toMillis(time))
5656

5757
test("datetime function current_date") {
58-
val d0 = DateTimeUtils.millisToDays(System.currentTimeMillis(), TimeZoneGMT)
58+
val d0 = DateTimeUtils.millisToDays(System.currentTimeMillis(), ZoneOffset.UTC)
5959
val cd = CurrentDate(gmtId).eval(EmptyRow).asInstanceOf[Int]
60-
val d1 = DateTimeUtils.millisToDays(System.currentTimeMillis(), TimeZoneGMT)
60+
val d1 = DateTimeUtils.millisToDays(System.currentTimeMillis(), ZoneOffset.UTC)
6161
assert(d0 <= cd && cd <= d1 && d1 - d0 <= 1)
6262

6363
val cdjst = CurrentDate(jstId).eval(EmptyRow).asInstanceOf[Int]
@@ -772,14 +772,15 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
772772
1000L)
773773
checkEvaluation(
774774
UnixTimestamp(Literal(date1), Literal("yyyy-MM-dd HH:mm:ss"), timeZoneId),
775-
MILLISECONDS.toSeconds(DateTimeUtils.daysToMillis(DateTimeUtils.fromJavaDate(date1), tz)))
775+
MILLISECONDS.toSeconds(
776+
DateTimeUtils.daysToMillis(DateTimeUtils.fromJavaDate(date1), tz.toZoneId)))
776777
checkEvaluation(
777778
UnixTimestamp(Literal(sdf2.format(new Timestamp(-1000000))), Literal(fmt2), timeZoneId),
778779
-1000L)
779780
checkEvaluation(UnixTimestamp(
780781
Literal(sdf3.format(Date.valueOf("2015-07-24"))), Literal(fmt3), timeZoneId),
781782
MILLISECONDS.toSeconds(DateTimeUtils.daysToMillis(
782-
DateTimeUtils.fromJavaDate(Date.valueOf("2015-07-24")), tz)))
783+
DateTimeUtils.fromJavaDate(Date.valueOf("2015-07-24")), tz.toZoneId)))
783784
val t1 = UnixTimestamp(
784785
CurrentTimestamp(), Literal("yyyy-MM-dd HH:mm:ss")).eval().asInstanceOf[Long]
785786
val t2 = UnixTimestamp(
@@ -794,7 +795,8 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
794795
null)
795796
checkEvaluation(
796797
UnixTimestamp(Literal(date1), Literal.create(null, StringType), timeZoneId),
797-
MILLISECONDS.toSeconds(DateTimeUtils.daysToMillis(DateTimeUtils.fromJavaDate(date1), tz)))
798+
MILLISECONDS.toSeconds(
799+
DateTimeUtils.daysToMillis(DateTimeUtils.fromJavaDate(date1), tz.toZoneId)))
798800
checkEvaluation(
799801
UnixTimestamp(Literal("2015-07-24"), Literal("not a valid format"), timeZoneId), null)
800802
}
@@ -827,14 +829,15 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
827829
1000L)
828830
checkEvaluation(
829831
ToUnixTimestamp(Literal(date1), Literal(fmt1), timeZoneId),
830-
MILLISECONDS.toSeconds(DateTimeUtils.daysToMillis(DateTimeUtils.fromJavaDate(date1), tz)))
832+
MILLISECONDS.toSeconds(
833+
DateTimeUtils.daysToMillis(DateTimeUtils.fromJavaDate(date1), tz.toZoneId)))
831834
checkEvaluation(
832835
ToUnixTimestamp(Literal(sdf2.format(new Timestamp(-1000000))), Literal(fmt2), timeZoneId),
833836
-1000L)
834837
checkEvaluation(ToUnixTimestamp(
835838
Literal(sdf3.format(Date.valueOf("2015-07-24"))), Literal(fmt3), timeZoneId),
836839
MILLISECONDS.toSeconds(DateTimeUtils.daysToMillis(
837-
DateTimeUtils.fromJavaDate(Date.valueOf("2015-07-24")), tz)))
840+
DateTimeUtils.fromJavaDate(Date.valueOf("2015-07-24")), tz.toZoneId)))
838841
val t1 = ToUnixTimestamp(
839842
CurrentTimestamp(), Literal(fmt1)).eval().asInstanceOf[Long]
840843
val t2 = ToUnixTimestamp(
@@ -848,7 +851,8 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
848851
null)
849852
checkEvaluation(ToUnixTimestamp(
850853
Literal(date1), Literal.create(null, StringType), timeZoneId),
851-
MILLISECONDS.toSeconds(DateTimeUtils.daysToMillis(DateTimeUtils.fromJavaDate(date1), tz)))
854+
MILLISECONDS.toSeconds(
855+
DateTimeUtils.daysToMillis(DateTimeUtils.fromJavaDate(date1), tz.toZoneId)))
852856
checkEvaluation(
853857
ToUnixTimestamp(Literal("2015-07-24"), Literal("not a valid format"), timeZoneId), null)
854858

‎sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -413,22 +413,22 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
413413
test("monthsBetween") {
414414
val date1 = date(1997, 2, 28, 10, 30, 0)
415415
var date2 = date(1996, 10, 30)
416-
assert(monthsBetween(date1, date2, true, TimeZoneUTC) === 3.94959677)
417-
assert(monthsBetween(date1, date2, false, TimeZoneUTC) === 3.9495967741935485)
416+
assert(monthsBetween(date1, date2, true, ZoneOffset.UTC) === 3.94959677)
417+
assert(monthsBetween(date1, date2, false, ZoneOffset.UTC) === 3.9495967741935485)
418418
Seq(true, false).foreach { roundOff =>
419419
date2 = date(2000, 2, 28)
420-
assert(monthsBetween(date1, date2, roundOff, TimeZoneUTC) === -36)
420+
assert(monthsBetween(date1, date2, roundOff, ZoneOffset.UTC) === -36)
421421
date2 = date(2000, 2, 29)
422-
assert(monthsBetween(date1, date2, roundOff, TimeZoneUTC) === -36)
422+
assert(monthsBetween(date1, date2, roundOff, ZoneOffset.UTC) === -36)
423423
date2 = date(1996, 3, 31)
424-
assert(monthsBetween(date1, date2, roundOff, TimeZoneUTC) === 11)
424+
assert(monthsBetween(date1, date2, roundOff, ZoneOffset.UTC) === 11)
425425
}
426426

427427
val date3 = date(2000, 2, 28, 16, tz = TimeZonePST)
428428
val date4 = date(1997, 2, 28, 16, tz = TimeZonePST)
429-
assert(monthsBetween(date3, date4, true, TimeZonePST) === 36.0)
430-
assert(monthsBetween(date3, date4, true, TimeZoneGMT) === 35.90322581)
431-
assert(monthsBetween(date3, date4, false, TimeZoneGMT) === 35.903225806451616)
429+
assert(monthsBetween(date3, date4, true, TimeZonePST.toZoneId) === 36.0)
430+
assert(monthsBetween(date3, date4, true, ZoneOffset.UTC) === 35.90322581)
431+
assert(monthsBetween(date3, date4, false, ZoneOffset.UTC) === 35.903225806451616)
432432
}
433433

434434
test("from UTC timestamp") {
@@ -571,15 +571,15 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
571571

572572
test("daysToMillis and millisToDays") {
573573
val input = TimeUnit.MICROSECONDS.toMillis(date(2015, 12, 31, 16, tz = TimeZonePST))
574-
assert(millisToDays(input, TimeZonePST) === 16800)
575-
assert(millisToDays(input, TimeZoneGMT) === 16801)
576-
assert(millisToDays(-1 * MILLIS_PER_DAY + 1, TimeZoneGMT) == -1)
574+
assert(millisToDays(input, TimeZonePST.toZoneId) === 16800)
575+
assert(millisToDays(input, ZoneOffset.UTC) === 16801)
576+
assert(millisToDays(-1 * MILLIS_PER_DAY + 1, ZoneOffset.UTC) == -1)
577577

578578
var expected = TimeUnit.MICROSECONDS.toMillis(date(2015, 12, 31, tz = TimeZonePST))
579-
assert(daysToMillis(16800, TimeZonePST) === expected)
579+
assert(daysToMillis(16800, TimeZonePST.toZoneId) === expected)
580580

581581
expected = TimeUnit.MICROSECONDS.toMillis(date(2015, 12, 31, tz = TimeZoneGMT))
582-
assert(daysToMillis(16800, TimeZoneGMT) === expected)
582+
assert(daysToMillis(16800, ZoneOffset.UTC) === expected)
583583

584584
// There are some days are skipped entirely in some timezone, skip them here.
585585
val skipped_days = Map[String, Set[Int]](
@@ -594,7 +594,7 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper {
594594
val skipped = skipped_days.getOrElse(tz.getID, Set.empty)
595595
(-20000 to 20000).foreach { d =>
596596
if (!skipped.contains(d)) {
597-
assert(millisToDays(daysToMillis(d, tz), tz) === d,
597+
assert(millisToDays(daysToMillis(d, tz.toZoneId), tz.toZoneId) === d,
598598
s"Round trip of ${d} did not work in tz ${tz}")
599599
}
600600
}

0 commit comments

Comments
 (0)