Skip to content

Commit c30982d

Browse files
pzzsmarmbrus
authored andcommitted
[SPARK-7437] [SQL] Fold "literal in (item1, item2, ..., literal, ...)" into true or false directly
SQL ``` select key from src where 3 in (4, 5); ``` Before ``` == Optimized Logical Plan == Project [key#12] Filter 3 INSET (5,4) MetastoreRelation default, src, None ``` After ``` == Optimized Logical Plan == LocalRelation [key#228], [] ``` Author: Zhongshuai Pei <799203320@qq.com> Author: DoingDone9 <799203320@qq.com> Closes apache#5972 from DoingDone9/InToFalse and squashes the following commits: 4c722a2 [Zhongshuai Pei] Update predicates.scala abe2bbb [Zhongshuai Pei] Update Optimizer.scala fa461a5 [Zhongshuai Pei] Update Optimizer.scala e34c28a [Zhongshuai Pei] Update predicates.scala 24739bd [Zhongshuai Pei] Update ConstantFoldingSuite.scala f4dbf50 [Zhongshuai Pei] Update ConstantFoldingSuite.scala 35ceb7a [Zhongshuai Pei] Update Optimizer.scala 36c194e [Zhongshuai Pei] Update Optimizer.scala 2e8f6ca [Zhongshuai Pei] Update Optimizer.scala 14952e2 [Zhongshuai Pei] Merge pull request apache#13 from apache/master f03fe7f [Zhongshuai Pei] Merge pull request apache#12 from apache/master f12fa50 [Zhongshuai Pei] Merge pull request apache#10 from apache/master f61210c [Zhongshuai Pei] Merge pull request apache#9 from apache/master 34b1a9a [Zhongshuai Pei] Merge pull request apache#8 from apache/master 802261c [DoingDone9] Merge pull request apache#7 from apache/master d00303b [DoingDone9] Merge pull request apache#6 from apache/master 98b134f [DoingDone9] Merge pull request apache#5 from apache/master 161cae3 [DoingDone9] Merge pull request #4 from apache/master c87e8b6 [DoingDone9] Merge pull request #3 from apache/master cb1852d [DoingDone9] Merge pull request #2 from apache/master c3f046f [DoingDone9] Merge pull request #1 from apache/master (cherry picked from commit 4b5e1fe) Signed-off-by: Michael Armbrust <michael@databricks.com>
1 parent 1a664a0 commit c30982d

File tree

3 files changed

+36
-2
lines changed

3 files changed

+36
-2
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,7 @@ case class InSet(value: Expression, hset: Set[Any])
110110

111111
override def children: Seq[Expression] = value :: Nil
112112

113+
override def foldable: Boolean = value.foldable
113114
override def nullable: Boolean = true // TODO: Figure out correct nullability semantics of IN.
114115
override def toString: String = s"$value INSET ${hset.mkString("(", ",", ")")}"
115116

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,13 +46,13 @@ object DefaultOptimizer extends Optimizer {
4646
CombineLimits) ::
4747
Batch("ConstantFolding", FixedPoint(100),
4848
NullPropagation,
49+
OptimizeIn,
4950
ConstantFolding,
5051
LikeSimplification,
5152
BooleanSimplification,
5253
SimplifyFilters,
5354
SimplifyCasts,
54-
SimplifyCaseConversionExpressions,
55-
OptimizeIn) ::
55+
SimplifyCaseConversionExpressions) ::
5656
Batch("Decimal Optimizations", FixedPoint(100),
5757
DecimalAggregates) ::
5858
Batch("LocalRelation", FixedPoint(100),

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ConstantFoldingSuite.scala

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ class ConstantFoldingSuite extends PlanTest {
3535
Batch("AnalysisNodes", Once,
3636
EliminateSubQueries) ::
3737
Batch("ConstantFolding", Once,
38+
OptimizeIn,
3839
ConstantFolding,
3940
BooleanSimplification) :: Nil
4041
}
@@ -247,4 +248,36 @@ class ConstantFoldingSuite extends PlanTest {
247248

248249
comparePlans(optimized, correctAnswer)
249250
}
251+
252+
test("Constant folding test: Fold In(v, list) into true or false") {
253+
var originalQuery =
254+
testRelation
255+
.select('a)
256+
.where(In(Literal(1), Seq(Literal(1), Literal(2))))
257+
258+
var optimized = Optimize.execute(originalQuery.analyze)
259+
260+
var correctAnswer =
261+
testRelation
262+
.select('a)
263+
.where(Literal(true))
264+
.analyze
265+
266+
comparePlans(optimized, correctAnswer)
267+
268+
originalQuery =
269+
testRelation
270+
.select('a)
271+
.where(In(Literal(1), Seq(Literal(1), 'a.attr)))
272+
273+
optimized = Optimize.execute(originalQuery.analyze)
274+
275+
correctAnswer =
276+
testRelation
277+
.select('a)
278+
.where(Literal(true))
279+
.analyze
280+
281+
comparePlans(optimized, correctAnswer)
282+
}
250283
}

0 commit comments

Comments
 (0)