Skip to content

Commit fcef14a

Browse files
committed
Address review comments
1 parent 66d00a3 commit fcef14a

File tree

5 files changed

+12
-25
lines changed

5 files changed

+12
-25
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -383,12 +383,6 @@ case class InSet(child: Expression, hset: Set[Any]) extends UnaryExpression with
383383
}
384384
}
385385

386-
override def sql: String = {
387-
val valueSQL = child.sql
388-
val listSQL = hset.toSeq.map(Literal(_).sql).mkString(", ")
389-
s"($valueSQL IN ($listSQL))"
390-
}
391-
392386
private def canBeComputedUsingSwitch: Boolean = child.dataType match {
393387
case ByteType | ShortType | IntegerType | DateType => true
394388
case _ => false
@@ -409,6 +403,8 @@ case class InSet(child: Expression, hset: Set[Any]) extends UnaryExpression with
409403
})
410404
}
411405

406+
// spark.sql.optimizer.inSetSwitchThreshold has an appropriate upper limit,
407+
// so the code size should not exceed 64KB
412408
private def genCodeWithSwitch(ctx: CodegenContext, ev: ExprCode): ExprCode = {
413409
val caseValuesGen = hset.filter(_ != null).map(Literal(_).genCode(ctx))
414410
val valueGen = child.genCode(ctx)
@@ -427,13 +423,19 @@ case class InSet(child: Expression, hset: Set[Any]) extends UnaryExpression with
427423
${CodeGenerator.JAVA_BOOLEAN} ${ev.value} = false;
428424
if (!${valueGen.isNull}) {
429425
switch (${valueGen.value}) {
430-
${caseBranches.mkString("")}
426+
${caseBranches.mkString("\n")}
431427
default:
432428
${ev.isNull} = $hasNull;
433429
}
434430
}
435431
""")
436432
}
433+
434+
override def sql: String = {
435+
val valueSQL = child.sql
436+
val listSQL = hset.toSeq.map(Literal(_).sql).mkString(", ")
437+
s"($valueSQL IN ($listSQL))"
438+
}
437439
}
438440

439441
@ExpressionDescription(

sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -178,7 +178,7 @@ object SQLConf {
178178
"switch statements. This is applicable only to bytes, shorts, ints, dates.")
179179
.intConf
180180
.checkValue(threshold => threshold >= 0 && threshold <= 600, "The max set size " +
181-
"for using switch statements in InSet must be positive and less than or equal to 600")
181+
"for using switch statements in InSet must be non-negative and less than or equal to 600")
182182
.createWithDefault(400)
183183

184184
val OPTIMIZER_PLAN_CHANGE_LOG_LEVEL = buildConf("spark.sql.optimizer.planChangeLog.level")

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/PredicateSuite.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -242,7 +242,7 @@ class PredicateSuite extends SparkFunSuite with ExpressionEvalHelper {
242242
}
243243
}
244244

245-
test("SPARK-26205: Optimize InSet for bytes, shorts, ints, dates using switch statements") {
245+
test("switch statements in InSet for bytes, shorts, ints, dates") {
246246
val byteValues = Set[Any](1.toByte, 2.toByte, Byte.MinValue, Byte.MaxValue)
247247
val shortValues = Set[Any](-10.toShort, 20.toShort, Short.MinValue, Short.MaxValue)
248248
val intValues = Set[Any](20, -100, 30, Int.MinValue, Int.MaxValue)

sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -415,7 +415,7 @@ class ColumnExpressionSuite extends QueryTest with SharedSQLContext {
415415
}
416416
}
417417

418-
test("SPARK-26205: Optimize InSet for bytes, shorts, ints, dates") {
418+
test("IN/INSET with bytes, shorts, ints, dates") {
419419
def check(): Unit = {
420420
val values = Seq(
421421
(Byte.MinValue, Some(Short.MinValue), Int.MinValue, Date.valueOf("2017-01-01")),

sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -310,19 +310,4 @@ class SQLConfSuite extends QueryTest with SharedSQLContext {
310310
SQLConf.unregister(fallback)
311311
}
312312

313-
test("SPARK-26205: spark.sql.optimizer.inSetSwitchThreshold must be handled correctly") {
314-
spark.sessionState.conf.clear()
315-
316-
assert(spark.conf.get(SQLConf.OPTIMIZER_INSET_SWITCH_THRESHOLD) == 400)
317-
spark.conf.set(SQLConf.OPTIMIZER_INSET_SWITCH_THRESHOLD.key, 50)
318-
assert(spark.conf.get(SQLConf.OPTIMIZER_INSET_SWITCH_THRESHOLD) == 50)
319-
intercept[IllegalArgumentException] {
320-
spark.conf.set(SQLConf.OPTIMIZER_INSET_SWITCH_THRESHOLD.key, -1)
321-
}
322-
intercept[IllegalArgumentException] {
323-
spark.conf.set(SQLConf.OPTIMIZER_INSET_SWITCH_THRESHOLD.key, 601)
324-
}
325-
326-
spark.sessionState.conf.clear()
327-
}
328313
}

0 commit comments

Comments
 (0)