Skip to content

Commit d2739af

Browse files
committed
Address comments from Marco
1 parent 87f0f50 commit d2739af

File tree

1 file changed

+6
-8
lines changed
  • sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer

1 file changed

+6
-8
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/joins.scala

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,7 @@ object EliminateOuterJoin extends Rule[LogicalPlan] with PredicateHelper {
157157
/**
158158
* PythonUDF in join condition can not be evaluated, this rule will detect the PythonUDF
159159
* and pull them out from join condition. For python udf accessing attributes from only one side,
160-
* they would be pushed down by operation push down rules. If not(e.g. user disables filter push
160+
* they are pushed down by operation push down rules. If not (e.g. user disables filter push
161161
* down rules), we need to pull them out in this rule too.
162162
*/
163163
object PullOutPythonUDFInJoinCondition extends Rule[LogicalPlan] with PredicateHelper {
@@ -178,14 +178,12 @@ object PullOutPythonUDFInJoinCondition extends Rule[LogicalPlan] with PredicateH
178178
s" $joinType is not supported.")
179179
}
180180
// If condition expression contains python udf, it will be moved out from
181-
// the new join conditions. If join condition has python udf only, it will be turned
182-
// to cross join and the crossJoinEnable will be checked in CheckCartesianProducts.
181+
// the new join conditions.
183182
val (udf, rest) =
184-
condition.map(splitConjunctivePredicates).get.partition(hasPythonUDF)
183+
splitConjunctivePredicates(condition.get).partition(hasPythonUDF)
185184
val newCondition = if (rest.isEmpty) {
186-
logWarning(s"The join condition:$condition of the join plan contains " +
187-
"PythonUDF only, it will be moved out and the join plan will be turned to cross " +
188-
s"join. This plan shows below:\n $j")
185+
logWarning(s"The join condition:$condition of the join plan contains PythonUDF only," +
186+
s" it will be moved out and the join plan will be turned to cross join.")
189187
None
190188
} else {
191189
Some(rest.reduceLeft(And))
@@ -196,7 +194,7 @@ object PullOutPythonUDFInJoinCondition extends Rule[LogicalPlan] with PredicateH
196194
case LeftSemi =>
197195
Project(
198196
j.left.output.map(_.toAttribute),
199-
Filter(udf.reduceLeft(And), newJoin.copy(joinType = Inner)))
197+
Filter(udf.reduceLeft(And), newJoin.copy(joinType = Inner)))
200198
case _ =>
201199
throw new AnalysisException("Using PythonUDF in join condition of join type" +
202200
s" $joinType is not supported.")

0 commit comments

Comments
 (0)