diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java
index 6a0933b0c6c0a7..96dae05b7fd045 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java
@@ -297,6 +297,7 @@ public void createScanRangeLocations() throws UserException {
                 location.setServer(new TNetworkAddress(backend.getHost(), backend.getBePort()));
                 curLocations.addToLocations(location);
                 scanRangeLocations.add(curLocations);
+                scanBackendIds.add(backendId);
                 return;
             }
         }
@@ -346,6 +347,7 @@ public void createScanRangeLocations() throws UserException {
                 // However, even one ScanNode instance can provide maximum scanning concurrency.
                 scanRangeLocations.add(curLocations);
                 setLocationPropertiesIfNecessary(backend, locationType, locationProperties);
+                scanBackendIds.add(backend.getId());
             }
         } else {
             List<Split> inputSplits = getSplits();
@@ -363,6 +365,7 @@ public void createScanRangeLocations() throws UserException {
                     scanRangeLocations.add(splitToScanRange(backend, locationProperties, split, pathPartitionKeys));
                     totalFileSize += split.getLength();
                 }
+                scanBackendIds.add(backend.getId());
             }
         }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
index afdffc748c01a6..41c055062e3e9b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
@@ -192,8 +192,6 @@ public class OlapScanNode extends ScanNode {
     private Set<Long> sampleTabletIds = Sets.newHashSet();
     private TableSample tableSample;
 
-    private HashSet<Long> scanBackendIds = new HashSet<>();
-
     private Map<Long, Integer> tabletId2BucketSeq = Maps.newHashMap();
     // a bucket seq may map to many tablets, and each tablet has a
     // TScanRangeLocations.
@@ -1919,9 +1917,4 @@ public boolean pushDownAggNoGroupingCheckCol(FunctionCallExpr aggExpr, Column co
     public int getScanRangeNum() {
         return getScanTabletIds().size();
     }
-
-    @Override
-    public int numScanBackends() {
-        return scanBackendIds.size();
-    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java
index 53aa0d5b24df5d..a2583868346704 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java
@@ -111,6 +111,11 @@ public abstract class ScanNode extends PlanNode implements SplitGenerator {
 
     protected TableSnapshot tableSnapshot;
 
+    // Save the id of backends which this scan node will be executed on.
+    // This is also important for local shuffle logic.
+    // Now only OlapScanNode and FileQueryScanNode implement this.
+    protected HashSet<Long> scanBackendIds = new HashSet<>();
+
     public ScanNode(PlanNodeId id, TupleDescriptor desc, String planNodeName, StatisticalType statisticalType) {
         super(id, desc.getId().asList(), planNodeName, statisticalType);
         this.desc = desc;
@@ -732,7 +737,7 @@ && getScanRangeNum()
     }
 
     public int numScanBackends() {
-        return 0;
+        return scanBackendIds.size();
     }
 
     public int getScanRangeNum() {