remove splitTasks (pingcap#875)

wanfangdata · Jun 28, 2019 · 2aa7320 · 2aa7320
1 parent 97512b6
commit 2aa7320
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 27 deletions.
diff --git a/core/src/main/scala/org/apache/spark/sql/execution/CoprocessorRDD.scala b/core/src/main/scala/org/apache/spark/sql/execution/CoprocessorRDD.scala
@@ -300,31 +300,6 @@ case class RegionTaskExec(child: SparkPlan,
         completionService.submit(task)
       }
 
-      // If one task's ranges list exceeds some threshold, we split it into two sub tasks and
-      // each has half of the original ranges.
-      def splitTasks(task: RegionTask): mutable.Seq[RegionTask] = {
-        val finalTasks = mutable.ListBuffer[RegionTask]()
-        val queue = mutable.Queue[RegionTask]()
-        queue += task
-        while (queue.nonEmpty) {
-          val front = queue.dequeue
-          if (isTaskRangeSizeInvalid(front)) {
-            // use (size + 1) / 2 here rather than size / 2
-            // to avoid extra single task generated by odd list
-            front.getRanges
-              .grouped((front.getRanges.size() + 1) / 2)
-              .foreach(range => {
-                queue += RegionTask.newInstance(front.getRegion, front.getStore, range)
-              })
-          } else {
-            // add all ranges satisfying task range size to final task list
-            finalTasks += front
-          }
-        }
-        logger.debug(s"Split $task into ${finalTasks.size} tasks.")
-        finalTasks
-      }
-
       def feedBatch(): TLongArrayList = {
         val handles = new array.TLongArrayList(512)
         while (handleIterator.hasNext &&
@@ -344,7 +319,7 @@ case class RegionTaskExec(child: SparkPlan,
 
           indexTasks.foreach { task =>
             val taskRange = task.getRanges
-            val tasks = splitTasks(task)
+            val tasks = Seq(task)
             numIndexScanTasks += tasks.size
 
             if (logger.isDebugEnabled) {

diff --git a/tikv-client/src/main/java/com/pingcap/tikv/operation/iterator/CoprocessIterator.java b/tikv-client/src/main/java/com/pingcap/tikv/operation/iterator/CoprocessIterator.java
@@ -35,7 +35,7 @@
 public abstract class CoprocessIterator<T> implements Iterator<T> {
   protected final TiSession session;
   protected final List<RegionTask> regionTasks;
-  protected DAGRequest dagRequest;
+  protected final DAGRequest dagRequest;
   protected final DataType[] handleTypes;
   //  protected final ExecutorCompletionService<Iterator<SelectResponse>> completionService;
   protected RowReader rowReader;