Spark: Do not pass numPartitions in 3.1 to let AQE size stages (apach…

…e#3203)
yabola · Oct 6, 2021 · adc6d15 · adc6d15
1 parent 94ba050
commit adc6d15
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/spark3/src/main/scala/org/apache/spark/sql/catalyst/utils/PlanUtils.scala b/spark3/src/main/scala/org/apache/spark/sql/catalyst/utils/PlanUtils.scala
@@ -61,7 +61,8 @@ object PlanUtils {
     if (Spark3VersionUtil.isSpark30) {
       repartitionByExpressionCtor.newInstance(partitionExpressions, child, Integer.valueOf(numPartitions))
     } else {
-      repartitionByExpressionCtor.newInstance(partitionExpressions, child, Some(numPartitions))
+      // Do not pass numPartitions because it is set automatically for AQE
+      repartitionByExpressionCtor.newInstance(partitionExpressions, child, None)
     }
   }
 }