Skip to content

Commit ee86c8a

Browse files
cloud-fancloud-fan
committed
make INTERVAL non-optional
1 parent e04309c commit ee86c8a

File tree

8 files changed

+36
-327
lines changed

8 files changed

+36
-327
lines changed

‎docs/sql-keywords.md

Lines changed: 6 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ license: |
2222
When `spark.sql.ansi.enabled` is true, Spark SQL will use the ANSI mode parser.
2323
In this mode, Spark SQL has two kinds of keywords:
2424
* Reserved keywords: Keywords that are reserved and can't be used as identifiers for table, view, column, function, alias, etc.
25-
* Non-reserved keywords: Keywords that have a special meaning only in particular contexts and can be used as identifiers in other contexts. For example, `SELECT 1 WEEK` is an interval literal, but WEEK can be used as identifiers in other places.
25+
* Non-reserved keywords: Keywords that have a special meaning only in particular contexts and can be used as identifiers in other contexts. For example, `EXPLAIN SELECT ...` is a command, but EXPLAIN can be used as identifiers in other places.
2626

2727
When the ANSI mode is disabled, Spark SQL has two kinds of keywords:
2828
* Non-reserved keywords: Same definition as the one when the ANSI mode enabled.
@@ -87,8 +87,7 @@ Below is a list of all the keywords in Spark SQL.
8787
<tr><td>DATA</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
8888
<tr><td>DATABASE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
8989
<tr><td>DATABASES</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
90-
<tr><td>DAY</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
91-
<tr><td>DAYS</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
90+
<tr><td>DAY</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
9291
<tr><td>DBPROPERTIES</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
9392
<tr><td>DEFINED</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
9493
<tr><td>DELETE</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
@@ -135,8 +134,7 @@ Below is a list of all the keywords in Spark SQL.
135134
<tr><td>GROUP</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
136135
<tr><td>GROUPING</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
137136
<tr><td>HAVING</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
138-
<tr><td>HOUR</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
139-
<tr><td>HOURS</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
137+
<tr><td>HOUR</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
140138
<tr><td>IF</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
141139
<tr><td>IGNORE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
142140
<tr><td>IMPORT</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
@@ -174,15 +172,9 @@ Below is a list of all the keywords in Spark SQL.
174172
<tr><td>MAP</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
175173
<tr><td>MATCHED</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
176174
<tr><td>MERGE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
177-
<tr><td>MICROSECOND</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
178-
<tr><td>MICROSECONDS</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
179-
<tr><td>MILLISECOND</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
180-
<tr><td>MILLISECONDS</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
181175
<tr><td>MINUS</td><td>reserved</td><td>strict-non-reserved</td><td>non-reserved</td></tr>
182176
<tr><td>MINUTE</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
183-
<tr><td>MINUTES</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
184-
<tr><td>MONTH</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
185-
<tr><td>MONTHS</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
177+
<tr><td>MONTH</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
186178
<tr><td>MSCK</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
187179
<tr><td>NAMESPACE</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
188180
<tr><td>NAMESPACES</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
@@ -241,8 +233,7 @@ Below is a list of all the keywords in Spark SQL.
241233
<tr><td>ROW</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
242234
<tr><td>ROWS</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
243235
<tr><td>SCHEMA</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
244-
<tr><td>SECOND</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
245-
<tr><td>SECONDS</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
236+
<tr><td>SECOND</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
246237
<tr><td>SELECT</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
247238
<tr><td>SEMI</td><td>reserved</td><td>strict-non-reserved</td><td>non-reserved</td></tr>
248239
<tr><td>SEPARATED</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
@@ -293,12 +284,9 @@ Below is a list of all the keywords in Spark SQL.
293284
<tr><td>USING</td><td>reserved</td><td>strict-non-reserved</td><td>reserved</td></tr>
294285
<tr><td>VALUES</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
295286
<tr><td>VIEW</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
296-
<tr><td>WEEK</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
297-
<tr><td>WEEKS</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
298287
<tr><td>WHEN</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
299288
<tr><td>WHERE</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
300289
<tr><td>WINDOW</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
301290
<tr><td>WITH</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
302-
<tr><td>YEAR</td><td>reserved</td><td>non-reserved</td><td>reserved</td></tr>
303-
<tr><td>YEARS</td><td>non-reserved</td><td>non-reserved</td><td>non-reserved</td></tr>
291+
<tr><td>YEAR</td><td>non-reserved</td><td>non-reserved</td><td>reserved</td></tr>
304292
</table>

‎sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4

Lines changed: 9 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -813,51 +813,29 @@ booleanValue
813813

814814
interval
815815
: INTERVAL (errorCapturingMultiUnitsInterval | errorCapturingUnitToUnitInterval)?
816-
| {SQL_standard_keyword_behavior}? (errorCapturingMultiUnitsInterval | errorCapturingUnitToUnitInterval)
817816
;
818817

819818
errorCapturingMultiUnitsInterval
820819
: multiUnitsInterval unitToUnitInterval?
821820
;
822821

823822
multiUnitsInterval
824-
: (intervalValue intervalUnit)+
823+
: (intervalValue identifier)+
825824
;
826825

827826
errorCapturingUnitToUnitInterval
828827
: body=unitToUnitInterval (error1=multiUnitsInterval | error2=unitToUnitInterval)?
829828
;
830829

831830
unitToUnitInterval
832-
: value=intervalValue from=intervalUnit TO to=intervalUnit
831+
: value=intervalValue from=identifier TO to=identifier
833832
;
834833

835834
intervalValue
836835
: (PLUS | MINUS)? (INTEGER_VALUE | DECIMAL_VALUE)
837836
| STRING
838837
;
839838

840-
intervalUnit
841-
: DAY
842-
| DAYS
843-
| HOUR
844-
| HOURS
845-
| MICROSECOND
846-
| MICROSECONDS
847-
| MILLISECOND
848-
| MILLISECONDS
849-
| MINUTE
850-
| MINUTES
851-
| MONTH
852-
| MONTHS
853-
| SECOND
854-
| SECONDS
855-
| WEEK
856-
| WEEKS
857-
| YEAR
858-
| YEARS
859-
;
860-
861839
colPosition
862840
: position=FIRST | position=AFTER afterCol=errorCapturingIdentifier
863841
;
@@ -991,7 +969,7 @@ number
991969
// function, alias, etc.
992970
// - Non-reserved keywords:
993971
// Keywords that have a special meaning only in particular contexts and can be used as
994-
// identifiers in other contexts. For example, `SELECT 1 WEEK` is an interval literal, but WEEK
972+
// identifiers in other contexts. For example, `EXPLAIN SELECT ...` is a command, but EXPLAIN
995973
// can be used as identifiers in other places.
996974
// You can find the full keywords list by searching "Start of the keywords list" in this file.
997975
// The non-reserved keywords are listed below. Keywords not in this list are reserved keywords.
@@ -1029,7 +1007,7 @@ ansiNonReserved
10291007
| DATA
10301008
| DATABASE
10311009
| DATABASES
1032-
| DAYS
1010+
| DAY
10331011
| DBPROPERTIES
10341012
| DEFINED
10351013
| DELETE
@@ -1060,7 +1038,7 @@ ansiNonReserved
10601038
| FUNCTIONS
10611039
| GLOBAL
10621040
| GROUPING
1063-
| HOURS
1041+
| HOUR
10641042
| IF
10651043
| IGNORE
10661044
| IMPORT
@@ -1089,12 +1067,8 @@ ansiNonReserved
10891067
| MAP
10901068
| MATCHED
10911069
| MERGE
1092-
| MICROSECOND
1093-
| MICROSECONDS
1094-
| MILLISECOND
1095-
| MILLISECONDS
1096-
| MINUTES
1097-
| MONTHS
1070+
| MINUTE
1071+
| MONTH
10981072
| MSCK
10991073
| NAMESPACE
11001074
| NAMESPACES
@@ -1141,7 +1115,7 @@ ansiNonReserved
11411115
| ROW
11421116
| ROWS
11431117
| SCHEMA
1144-
| SECONDS
1118+
| SECOND
11451119
| SEPARATED
11461120
| SERDE
11471121
| SERDEPROPERTIES
@@ -1179,10 +1153,8 @@ ansiNonReserved
11791153
| USE
11801154
| VALUES
11811155
| VIEW
1182-
| WEEK
1183-
| WEEKS
11841156
| WINDOW
1185-
| YEARS
1157+
| YEAR
11861158
;
11871159

11881160
// When `SQL_standard_keyword_behavior=false`, there are 2 kinds of keywords in Spark SQL.
@@ -1264,7 +1236,6 @@ nonReserved
12641236
| DATABASE
12651237
| DATABASES
12661238
| DAY
1267-
| DAYS
12681239
| DBPROPERTIES
12691240
| DEFINED
12701241
| DELETE
@@ -1310,7 +1281,6 @@ nonReserved
13101281
| GROUPING
13111282
| HAVING
13121283
| HOUR
1313-
| HOURS
13141284
| IF
13151285
| IGNORE
13161286
| IMPORT
@@ -1344,14 +1314,8 @@ nonReserved
13441314
| MAP
13451315
| MATCHED
13461316
| MERGE
1347-
| MICROSECOND
1348-
| MICROSECONDS
1349-
| MILLISECOND
1350-
| MILLISECONDS
13511317
| MINUTE
1352-
| MINUTES
13531318
| MONTH
1354-
| MONTHS
13551319
| MSCK
13561320
| NAMESPACE
13571321
| NAMESPACES
@@ -1408,7 +1372,6 @@ nonReserved
14081372
| ROWS
14091373
| SCHEMA
14101374
| SECOND
1411-
| SECONDS
14121375
| SELECT
14131376
| SEPARATED
14141377
| SERDE
@@ -1457,14 +1420,11 @@ nonReserved
14571420
| USER
14581421
| VALUES
14591422
| VIEW
1460-
| WEEK
1461-
| WEEKS
14621423
| WHEN
14631424
| WHERE
14641425
| WINDOW
14651426
| WITH
14661427
| YEAR
1467-
| YEARS
14681428
;
14691429

14701430
// NOTE: If you add a new token in the list below, you should update the list of keywords
@@ -1527,7 +1487,6 @@ DATA: 'DATA';
15271487
DATABASE: 'DATABASE';
15281488
DATABASES: 'DATABASES' | 'SCHEMAS';
15291489
DAY: 'DAY';
1530-
DAYS: 'DAYS';
15311490
DBPROPERTIES: 'DBPROPERTIES';
15321491
DEFINED: 'DEFINED';
15331492
DELETE: 'DELETE';
@@ -1574,7 +1533,6 @@ GROUP: 'GROUP';
15741533
GROUPING: 'GROUPING';
15751534
HAVING: 'HAVING';
15761535
HOUR: 'HOUR';
1577-
HOURS: 'HOURS';
15781536
IF: 'IF';
15791537
IGNORE: 'IGNORE';
15801538
IMPORT: 'IMPORT';
@@ -1612,14 +1570,8 @@ MACRO: 'MACRO';
16121570
MAP: 'MAP';
16131571
MATCHED: 'MATCHED';
16141572
MERGE: 'MERGE';
1615-
MICROSECOND: 'MICROSECOND';
1616-
MICROSECONDS: 'MICROSECONDS';
1617-
MILLISECOND: 'MILLISECOND';
1618-
MILLISECONDS: 'MILLISECONDS';
16191573
MINUTE: 'MINUTE';
1620-
MINUTES: 'MINUTES';
16211574
MONTH: 'MONTH';
1622-
MONTHS: 'MONTHS';
16231575
MSCK: 'MSCK';
16241576
NAMESPACE: 'NAMESPACE';
16251577
NAMESPACES: 'NAMESPACES';
@@ -1679,7 +1631,6 @@ ROW: 'ROW';
16791631
ROWS: 'ROWS';
16801632
SCHEMA: 'SCHEMA';
16811633
SECOND: 'SECOND';
1682-
SECONDS: 'SECONDS';
16831634
SELECT: 'SELECT';
16841635
SEMI: 'SEMI';
16851636
SEPARATED: 'SEPARATED';
@@ -1732,14 +1683,11 @@ USER: 'USER';
17321683
USING: 'USING';
17331684
VALUES: 'VALUES';
17341685
VIEW: 'VIEW';
1735-
WEEK: 'WEEK';
1736-
WEEKS: 'WEEKS';
17371686
WHEN: 'WHEN';
17381687
WHERE: 'WHERE';
17391688
WINDOW: 'WINDOW';
17401689
WITH: 'WITH';
17411690
YEAR: 'YEAR';
1742-
YEARS: 'YEARS';
17431691
//============================
17441692
// End of the keywords list
17451693
//============================

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2088,7 +2088,7 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging
20882088
*/
20892089
override def visitMultiUnitsInterval(ctx: MultiUnitsIntervalContext): CalendarInterval = {
20902090
withOrigin(ctx) {
2091-
val units = ctx.intervalUnit().asScala
2091+
val units = ctx.identifier().asScala
20922092
val values = ctx.intervalValue().asScala
20932093
try {
20942094
assert(units.length == values.length)

‎sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala

Lines changed: 0 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -645,11 +645,6 @@ class ExpressionParserSuite extends AnalysisTest {
645645
"-" -> UnaryMinus(expected)
646646
).foreach { case (sign, expectedLiteral) =>
647647
assertEqual(s"${sign}interval $intervalValue", expectedLiteral)
648-
649-
// SPARK-23264 Support interval values without INTERVAL clauses if ANSI SQL enabled
650-
withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") {
651-
assertEqual(intervalValue, expected)
652-
}
653648
}
654649
}
655650

@@ -732,34 +727,6 @@ class ExpressionParserSuite extends AnalysisTest {
732727
Literal(new CalendarInterval(3, 4, 22001000L)))
733728
}
734729

735-
test("SPARK-23264 Interval Compatibility tests") {
736-
def checkIntervals(intervalValue: String, expected: Literal): Unit = {
737-
withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") {
738-
assertEqual(intervalValue, expected)
739-
}
740-
741-
// Compatibility tests: If ANSI SQL disabled, `intervalValue` should be parsed as an alias
742-
withSQLConf(SQLConf.ANSI_ENABLED.key -> "false") {
743-
val aliases = defaultParser.parseExpression(intervalValue).collect {
744-
case a @ Alias(_: Literal, name)
745-
if intervalUnits.exists { unit => name.startsWith(unit.toString) } => a
746-
}
747-
assert(aliases.size === 1)
748-
}
749-
}
750-
val forms = Seq("", "s")
751-
val values = Seq("5", "1", "-11", "8")
752-
intervalUnits.foreach { unit =>
753-
forms.foreach { form =>
754-
values.foreach { value =>
755-
val expected = intervalLiteral(unit, value)
756-
checkIntervals(s"$value $unit$form", expected)
757-
checkIntervals(s"'$value' $unit$form", expected)
758-
}
759-
}
760-
}
761-
}
762-
763730
test("composed expressions") {
764731
assertEqual("1 + r.r As q", (Literal(1) + UnresolvedAttribute("r.r")).as("q"))
765732
assertEqual("1 - f('o', o(bar))", Literal(1) - 'f.function("o", 'o.function('bar)))

0 commit comments

Comments
 (0)