Skip to content

Commit eae0214

Browse files
srowenJackey Lee
srowen
authored and
Jackey Lee
committed
[SPARK-26503][CORE] Get rid of spark.sql.legacy.timeParser.enabled
## What changes were proposed in this pull request? Per discussion in apache#23391 (comment) this proposes to just remove the old pre-Spark-3 time parsing behavior. This is a rebase of apache#23411 ## How was this patch tested? Existing tests. Closes apache#23495 from srowen/SPARK-26503.2. Authored-by: Sean Owen <sean.owen@databricks.com> Signed-off-by: Sean Owen <sean.owen@databricks.com>
1 parent 97f8933 commit eae0214

File tree

6 files changed

+120
-336
lines changed

6 files changed

+120
-336
lines changed

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala‎

Lines changed: 1 addition & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,6 @@ package org.apache.spark.sql.catalyst.util
2020
import java.time.{Instant, ZoneId}
2121
import java.util.Locale
2222

23-
import scala.util.Try
24-
25-
import org.apache.commons.lang3.time.FastDateFormat
26-
27-
import org.apache.spark.sql.internal.SQLConf
28-
2923
sealed trait DateFormatter extends Serializable {
3024
def parse(s: String): Int // returns days since epoch
3125
def format(days: Int): String
@@ -56,43 +50,8 @@ class Iso8601DateFormatter(
5650
}
5751
}
5852

59-
class LegacyDateFormatter(pattern: String, locale: Locale) extends DateFormatter {
60-
@transient
61-
private lazy val format = FastDateFormat.getInstance(pattern, locale)
62-
63-
override def parse(s: String): Int = {
64-
val milliseconds = format.parse(s).getTime
65-
DateTimeUtils.millisToDays(milliseconds)
66-
}
67-
68-
override def format(days: Int): String = {
69-
val date = DateTimeUtils.toJavaDate(days)
70-
format.format(date)
71-
}
72-
}
73-
74-
class LegacyFallbackDateFormatter(
75-
pattern: String,
76-
locale: Locale) extends LegacyDateFormatter(pattern, locale) {
77-
override def parse(s: String): Int = {
78-
Try(super.parse(s)).orElse {
79-
// If it fails to parse, then tries the way used in 2.0 and 1.x for backwards
80-
// compatibility.
81-
Try(DateTimeUtils.millisToDays(DateTimeUtils.stringToTime(s).getTime))
82-
}.getOrElse {
83-
// In Spark 1.5.0, we store the data as number of days since epoch in string.
84-
// So, we just convert it to Int.
85-
s.toInt
86-
}
87-
}
88-
}
89-
9053
object DateFormatter {
9154
def apply(format: String, locale: Locale): DateFormatter = {
92-
if (SQLConf.get.legacyTimeParserEnabled) {
93-
new LegacyFallbackDateFormatter(format, locale)
94-
} else {
95-
new Iso8601DateFormatter(format, locale)
96-
}
55+
new Iso8601DateFormatter(format, locale)
9756
}
9857
}

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala‎

Lines changed: 1 addition & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,6 @@ import java.time.format.DateTimeParseException
2323
import java.time.temporal.TemporalQueries
2424
import java.util.{Locale, TimeZone}
2525

26-
import scala.util.Try
27-
28-
import org.apache.commons.lang3.time.FastDateFormat
29-
30-
import org.apache.spark.sql.internal.SQLConf
31-
3226
sealed trait TimestampFormatter extends Serializable {
3327
/**
3428
* Parses a timestamp in a string and converts it to microseconds.
@@ -79,37 +73,8 @@ class Iso8601TimestampFormatter(
7973
}
8074
}
8175

82-
class LegacyTimestampFormatter(
83-
pattern: String,
84-
timeZone: TimeZone,
85-
locale: Locale) extends TimestampFormatter {
86-
@transient
87-
private lazy val format = FastDateFormat.getInstance(pattern, timeZone, locale)
88-
89-
protected def toMillis(s: String): Long = format.parse(s).getTime
90-
91-
override def parse(s: String): Long = toMillis(s) * DateTimeUtils.MICROS_PER_MILLIS
92-
93-
override def format(us: Long): String = {
94-
format.format(DateTimeUtils.toJavaTimestamp(us))
95-
}
96-
}
97-
98-
class LegacyFallbackTimestampFormatter(
99-
pattern: String,
100-
timeZone: TimeZone,
101-
locale: Locale) extends LegacyTimestampFormatter(pattern, timeZone, locale) {
102-
override def toMillis(s: String): Long = {
103-
Try {super.toMillis(s)}.getOrElse(DateTimeUtils.stringToTime(s).getTime)
104-
}
105-
}
106-
10776
object TimestampFormatter {
10877
def apply(format: String, timeZone: TimeZone, locale: Locale): TimestampFormatter = {
109-
if (SQLConf.get.legacyTimeParserEnabled) {
110-
new LegacyFallbackTimestampFormatter(format, timeZone, locale)
111-
} else {
112-
new Iso8601TimestampFormatter(format, timeZone, locale)
113-
}
78+
new Iso8601TimestampFormatter(format, timeZone, locale)
11479
}
11580
}

‎sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala‎

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1625,13 +1625,6 @@ object SQLConf {
16251625
"a SparkConf entry.")
16261626
.booleanConf
16271627
.createWithDefault(true)
1628-
1629-
val LEGACY_TIME_PARSER_ENABLED = buildConf("spark.sql.legacy.timeParser.enabled")
1630-
.doc("When set to true, java.text.SimpleDateFormat is used for formatting and parsing " +
1631-
" dates/timestamps in a locale-sensitive manner. When set to false, classes from " +
1632-
"java.time.* packages are used for the same purpose.")
1633-
.booleanConf
1634-
.createWithDefault(false)
16351628
}
16361629

16371630
/**
@@ -2057,8 +2050,6 @@ class SQLConf extends Serializable with Logging {
20572050
def setCommandRejectsSparkCoreConfs: Boolean =
20582051
getConf(SQLConf.SET_COMMAND_REJECTS_SPARK_CORE_CONFS)
20592052

2060-
def legacyTimeParserEnabled: Boolean = getConf(SQLConf.LEGACY_TIME_PARSER_ENABLED)
2061-
20622053
/** ********************** SQLConf functionality methods ************ */
20632054

20642055
/** Set Spark SQL configuration properties. */

‎sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/json/JsonInferSchemaSuite.scala‎

Lines changed: 30 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ import com.fasterxml.jackson.core.JsonFactory
2121

2222
import org.apache.spark.SparkFunSuite
2323
import org.apache.spark.sql.catalyst.plans.SQLHelper
24-
import org.apache.spark.sql.internal.SQLConf
2524
import org.apache.spark.sql.types._
2625

2726
class JsonInferSchemaSuite extends SparkFunSuite with SQLHelper {
@@ -43,61 +42,45 @@ class JsonInferSchemaSuite extends SparkFunSuite with SQLHelper {
4342
}
4443

4544
test("inferring timestamp type") {
46-
Seq(true, false).foreach { legacyParser =>
47-
withSQLConf(SQLConf.LEGACY_TIME_PARSER_ENABLED.key -> legacyParser.toString) {
48-
checkTimestampType("yyyy", """{"a": "2018"}""")
49-
checkTimestampType("yyyy=MM", """{"a": "2018=12"}""")
50-
checkTimestampType("yyyy MM dd", """{"a": "2018 12 02"}""")
51-
checkTimestampType(
52-
"yyyy-MM-dd'T'HH:mm:ss.SSS",
53-
"""{"a": "2018-12-02T21:04:00.123"}""")
54-
checkTimestampType(
55-
"yyyy-MM-dd'T'HH:mm:ss.SSSSSSXXX",
56-
"""{"a": "2018-12-02T21:04:00.123567+01:00"}""")
57-
}
58-
}
45+
checkTimestampType("yyyy", """{"a": "2018"}""")
46+
checkTimestampType("yyyy=MM", """{"a": "2018=12"}""")
47+
checkTimestampType("yyyy MM dd", """{"a": "2018 12 02"}""")
48+
checkTimestampType(
49+
"yyyy-MM-dd'T'HH:mm:ss.SSS",
50+
"""{"a": "2018-12-02T21:04:00.123"}""")
51+
checkTimestampType(
52+
"yyyy-MM-dd'T'HH:mm:ss.SSSSSSXXX",
53+
"""{"a": "2018-12-02T21:04:00.123567+01:00"}""")
5954
}
6055

6156
test("prefer decimals over timestamps") {
62-
Seq(true, false).foreach { legacyParser =>
63-
withSQLConf(SQLConf.LEGACY_TIME_PARSER_ENABLED.key -> legacyParser.toString) {
64-
checkType(
65-
options = Map(
66-
"prefersDecimal" -> "true",
67-
"timestampFormat" -> "yyyyMMdd.HHmmssSSS"
68-
),
69-
json = """{"a": "20181202.210400123"}""",
70-
dt = DecimalType(17, 9)
71-
)
72-
}
73-
}
57+
checkType(
58+
options = Map(
59+
"prefersDecimal" -> "true",
60+
"timestampFormat" -> "yyyyMMdd.HHmmssSSS"
61+
),
62+
json = """{"a": "20181202.210400123"}""",
63+
dt = DecimalType(17, 9)
64+
)
7465
}
7566

7667
test("skip decimal type inferring") {
77-
Seq(true, false).foreach { legacyParser =>
78-
withSQLConf(SQLConf.LEGACY_TIME_PARSER_ENABLED.key -> legacyParser.toString) {
79-
checkType(
80-
options = Map(
81-
"prefersDecimal" -> "false",
82-
"timestampFormat" -> "yyyyMMdd.HHmmssSSS"
83-
),
84-
json = """{"a": "20181202.210400123"}""",
85-
dt = TimestampType
86-
)
87-
}
88-
}
68+
checkType(
69+
options = Map(
70+
"prefersDecimal" -> "false",
71+
"timestampFormat" -> "yyyyMMdd.HHmmssSSS"
72+
),
73+
json = """{"a": "20181202.210400123"}""",
74+
dt = TimestampType
75+
)
8976
}
9077

9178
test("fallback to string type") {
92-
Seq(true, false).foreach { legacyParser =>
93-
withSQLConf(SQLConf.LEGACY_TIME_PARSER_ENABLED.key -> legacyParser.toString) {
94-
checkType(
95-
options = Map("timestampFormat" -> "yyyy,MM,dd.HHmmssSSS"),
96-
json = """{"a": "20181202.210400123"}""",
97-
dt = StringType
98-
)
99-
}
100-
}
79+
checkType(
80+
options = Map("timestampFormat" -> "yyyy,MM,dd.HHmmssSSS"),
81+
json = """{"a": "20181202.210400123"}""",
82+
dt = StringType
83+
)
10184
}
10285

10386
test("disable timestamp inferring") {

0 commit comments

Comments
 (0)