Fuse AllReduce Operator

paddle/fluid/framework/details/CMakeLists.txt

@@ -31,8 +31,15 @@ cc_library(fuse_vars_op_handle SRCS fuse_vars_op_handle.cc DEPS op_handle_base s
 cc_library(multi_devices_graph_builder SRCS multi_devices_graph_builder.cc DEPS ssa_graph_builder computation_op_handle
         scale_loss_grad_op_handle rpc_op_handle ${multi_devices_graph_builder_deps} reduce_op_handle broadcast_op_handle)
 
+if(WITH_GPU)
+    nv_library(fuse_all_reduce_graph_builder SRCS fuse_all_reduce_graph_builder.cc
+            DEPS ssa_graph_builder nccl_all_reduce_op_handle fuse_vars_op_handle)
+    set(graph_builder_factory_deps fuse_all_reduce_graph_builder)
+else()
+    set(graph_builder_factory_deps)
+endif()
 
-cc_library(ssa_graph_builder_factory SRCS ssa_graph_builder_factory.cc DEPS multi_devices_graph_builder ssa_graph_printer ssa_graph_checker)
+cc_library(ssa_graph_builder_factory SRCS ssa_graph_builder_factory.cc DEPS multi_devices_graph_builder ssa_graph_printer ssa_graph_checker ${graph_builder_factory_deps})
 
 cc_library(ssa_graph_executor SRCS ssa_graph_executor.cc DEPS ssa_graph framework_proto)
 cc_library(threaded_ssa_graph_executor SRCS threaded_ssa_graph_executor.cc DEPS fetch_op_handle ssa_graph_executor scope

paddle/fluid/framework/details/build_strategy.h

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 #pragma once
+#include <stdint.h>
 
 #include <string>
 
@@ -21,7 +22,22 @@ namespace framework {
 namespace details {
 
 struct BuildStrategy {
-  enum class ReduceStrategy { kAllReduce = 0, kReduce = 1 };
+  enum class ReduceStrategy {
+    kAllReduce = 0x0000,
+    kReduce = 0x0001,
+
+    kOperationMask = 0x00FF,
+    kFusedBit = 0x0100,
+
+    kFusedAllReduce = kFusedBit | kAllReduce,
+    kFusedReduce = kFusedBit | kReduce,
+  };
+
+  ReduceStrategy ReduceOperation() const {
+    return static_cast<ReduceStrategy>(
+        static_cast<uint16_t>(reduce_) &
+        static_cast<uint16_t>(ReduceStrategy::kOperationMask));
+  }
 
   enum class GradientScaleStrategy {
     kCoeffNumDevice = 0,