Skip to content

Commit 01d899d

Browse files
committed
Revert "[SPARK-26576][SQL] Broadcast hint not applied to partitioned table"
This reverts commit 3c2c84c.
1 parent c698589 commit 01d899d

File tree

2 files changed

+4
-18
lines changed

2 files changed

+4
-18
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,9 @@ object PhysicalOperation extends PredicateHelper {
6969
val substitutedCondition = substitute(aliases)(condition)
7070
(fields, filters ++ splitConjunctivePredicates(substitutedCondition), other, aliases)
7171

72+
case BroadcastHint(child) =>
73+
collectProjectsAndFilters(child)
74+
7275
case other =>
7376
(None, Nil, other, Map.empty)
7477
}

sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruneFileSourcePartitionsSuite.scala

Lines changed: 1 addition & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -17,19 +17,14 @@
1717

1818
package org.apache.spark.sql.hive.execution
1919

20-
import org.scalatest.Matchers._
21-
2220
import org.apache.spark.sql.QueryTest
2321
import org.apache.spark.sql.catalyst.dsl.expressions._
2422
import org.apache.spark.sql.catalyst.dsl.plans._
25-
import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, Project, ResolvedHint}
23+
import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, Project}
2624
import org.apache.spark.sql.catalyst.rules.RuleExecutor
2725
import org.apache.spark.sql.execution.datasources.{CatalogFileIndex, HadoopFsRelation, LogicalRelation, PruneFileSourcePartitions}
2826
import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat
29-
import org.apache.spark.sql.execution.joins.BroadcastHashJoinExec
30-
import org.apache.spark.sql.functions.broadcast
3127
import org.apache.spark.sql.hive.test.TestHiveSingleton
32-
import org.apache.spark.sql.internal.SQLConf
3328
import org.apache.spark.sql.test.SQLTestUtils
3429
import org.apache.spark.sql.types.StructType
3530

@@ -71,16 +66,4 @@ class PruneFileSourcePartitionsSuite extends QueryTest with SQLTestUtils with Te
7166
}
7267
}
7368
}
74-
75-
test("SPARK-26576 Broadcast hint not applied to partitioned table") {
76-
withTable("tbl") {
77-
withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
78-
spark.range(10).selectExpr("id", "id % 3 as p").write.partitionBy("p").saveAsTable("tbl")
79-
val df = spark.table("tbl")
80-
val qe = df.join(broadcast(df), "p").queryExecution
81-
qe.optimizedPlan.collect { case _: ResolvedHint => } should have size 1
82-
qe.sparkPlan.collect { case j: BroadcastHashJoinExec => j } should have size 1
83-
}
84-
}
85-
}
8669
}

0 commit comments

Comments
 (0)