[apache#889] improvement: Modify default value of single buffer flush. (

apache#1039) ### What changes were proposed in this pull request? Modify default value of single buffer flush. ### Why are the changes needed? For release 0.8.0. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? exiting UTs. --------- Co-authored-by: leixianming <leixianming@didiglobal.com>
vinlee19 · Jul 26, 2023 · 73c1ba2 · 73c1ba2
1 parent 79c95b0
commit 73c1ba2
Show file tree

Hide file tree

Showing 5 changed files with 11 additions and 6 deletions.
diff --git a/conf/server.conf b/conf/server.conf
@@ -26,3 +26,5 @@ rss.server.flush.thread.alive 5
 rss.server.flush.localfile.threadPool.size 10
 rss.server.flush.hadoop.threadPool.size 10
 rss.server.disk.capacity 1t
+rss.server.single.buffer.flush.enabled true
+rss.server.single.buffer.flush.threshold 128m
diff --git a/docs/server_guide.md b/docs/server_guide.md
@@ -83,8 +83,8 @@ This document will introduce how to deploy Uniffle shuffle servers.
 | rss.storage.type                                        | -       | Supports MEMORY_LOCALFILE, MEMORY_HDFS, MEMORY_LOCALFILE_HDFS                                                                                                                                                                                                                                                                                                                                |
 | rss.server.flush.cold.storage.threshold.size            | 64M     | The threshold of data size for LOACALFILE and HADOOP if MEMORY_LOCALFILE_HDFS is used                                                                                                                                                                                                                                                                                                        |
 | rss.server.tags                                         | -       | The comma-separated list of tags to indicate the shuffle server's attributes. It will be used as the assignment basis for the coordinator                                                                                                                                                                                                                                                    |
-| rss.server.single.buffer.flush.enabled                  | false   | Whether single buffer flush when size exceeded rss.server.single.buffer.flush.threshold                                                                                                                                                                                                                                                                                                      |
-| rss.server.single.buffer.flush.threshold                | 64M     | The threshold of single shuffle buffer flush                                                                                                                                                                                                                                                                                                                                                 |
+| rss.server.single.buffer.flush.enabled                  | true    | Whether single buffer flush when size exceeded rss.server.single.buffer.flush.threshold                                                                                                                                                                                                                                                                                                      |
+| rss.server.single.buffer.flush.threshold                | 128M    | The threshold of single shuffle buffer flush                                                                                                                                                                                                                                                                                                                                                 |
 | rss.server.disk.capacity                                | -1      | Disk capacity that shuffle server can use. If negative, it will use disk whole space * ratio                                                                                                                                                                                                                                                                                                 |
 | rss.server.disk.capacity.ratio                          | 0.9     | When `rss.server.disk.capacity` is negative, disk whole space * ratio is used                                                                                                                                                                                                                                                                                                                |
 | rss.server.multistorage.fallback.strategy.class         | -       | The fallback strategy for `MEMORY_LOCALFILE_HDFS`. Support `org.apache.uniffle.server.storage.RotateStorageManagerFallbackStrategy`,`org.apache.uniffle.server.storage.LocalStorageManagerFallbackStrategy` and `org.apache.uniffle.server.storage.HadoopStorageManagerFallbackStrategy`. If not set, `org.apache.uniffle.server.storage.HadoopStorageManagerFallbackStrategy` will be used. |

diff --git a/server/src/main/java/org/apache/uniffle/server/ShuffleServerConf.java b/server/src/main/java/org/apache/uniffle/server/ShuffleServerConf.java
@@ -339,14 +339,14 @@ public class ShuffleServerConf extends RssBaseConf {
   public static final ConfigOption<Boolean> SINGLE_BUFFER_FLUSH_ENABLED =
       ConfigOptions.key("rss.server.single.buffer.flush.enabled")
           .booleanType()
-          .defaultValue(false)
+          .defaultValue(true)
           .withDescription(
               "Whether single buffer flush when size exceeded rss.server.single.buffer.flush.threshold");
 
   public static final ConfigOption<Long> SINGLE_BUFFER_FLUSH_THRESHOLD =
       ConfigOptions.key("rss.server.single.buffer.flush.threshold")
           .longType()
-          .defaultValue(64 * 1024 * 1024L)
+          .defaultValue(128 * 1024 * 1024L)
           .withDescription("The threshold of single shuffle buffer flush");
 
   public static final ConfigOption<Long> STORAGEMANAGER_CACHE_TIMEOUT =

diff --git a/server/src/main/java/org/apache/uniffle/server/buffer/ShuffleBufferManager.java b/server/src/main/java/org/apache/uniffle/server/buffer/ShuffleBufferManager.java
@@ -109,8 +109,10 @@ public ShuffleBufferManager(ShuffleServerConf conf, ShuffleFlushManager shuffleF
                 / 100.0
                 * conf.get(ShuffleServerConf.SERVER_MEMORY_SHUFFLE_LOWWATERMARK_PERCENTAGE));
     this.bufferFlushEnabled = conf.getBoolean(ShuffleServerConf.SINGLE_BUFFER_FLUSH_ENABLED);
-    this.bufferFlushThreshold = conf.getLong(ShuffleServerConf.SINGLE_BUFFER_FLUSH_THRESHOLD);
-    this.shuffleFlushThreshold = conf.getLong(ShuffleServerConf.SERVER_SHUFFLE_FLUSH_THRESHOLD);
+    this.bufferFlushThreshold =
+        conf.getSizeAsBytes(ShuffleServerConf.SINGLE_BUFFER_FLUSH_THRESHOLD);
+    this.shuffleFlushThreshold =
+        conf.getSizeAsBytes(ShuffleServerConf.SERVER_SHUFFLE_FLUSH_THRESHOLD);
     this.hugePartitionSizeThreshold =
         conf.getSizeAsBytes(ShuffleServerConf.HUGE_PARTITION_SIZE_THRESHOLD);
     this.hugePartitionMemoryLimitSize =

diff --git a/server/src/test/java/org/apache/uniffle/server/buffer/ShuffleBufferManagerTest.java b/server/src/test/java/org/apache/uniffle/server/buffer/ShuffleBufferManagerTest.java
@@ -446,6 +446,7 @@ public void flushSingleBufferForHugePartitionTest(@TempDir File tmpDir) throws E
     shuffleConf.set(ShuffleServerConf.SERVER_BUFFER_CAPACITY, 200L);
     shuffleConf.set(ShuffleServerConf.HUGE_PARTITION_MEMORY_USAGE_LIMITATION_RATIO, 0.1);
     shuffleConf.set(ShuffleServerConf.HUGE_PARTITION_SIZE_THRESHOLD, 100L);
+    shuffleConf.set(ShuffleServerConf.SINGLE_BUFFER_FLUSH_ENABLED, false);
     shuffleConf.setSizeAsBytes(ShuffleServerConf.SINGLE_BUFFER_FLUSH_THRESHOLD, 64L);
 
     ShuffleServer mockShuffleServer = mock(ShuffleServer.class);