[hotfix] set return_outputs=False in examples and polish code (#5404)

* fix: simplify merge_batch

* fix: use return_outputs=False to eliminate extra memory consumption

* feat: add return_outputs warning

* style: remove `return_outputs=False` as it is the default value

Author: cwher

applications/ColossalMoE/train.py

@@ -238,7 +238,6 @@ def main():
                         lambda x, y: x.loss,
                         optimizer,
                         return_loss=True,
-                        return_outputs=True,
                     )
                     # Backward and optimize
                     if is_pp_last_stage:

colossalai/booster/plugin/hybrid_parallel_plugin.py

@@ -1183,6 +1183,9 @@ def execute_pipeline(
     ) -> dict:
         assert self.enable_pipeline_parallelism, "pipeline parallelism is not enabled"
 
+        if return_outputs:
+            warnings.warn("return_outputs may lead to significant extra memory consumption.")
+
         # Create a context for gradient synchronization based on the optimizer type.
         # If it's a HybridParallelZeroOptimizer, use optimizer.no_sync(); otherwise, use model.no_sync().
         # This is to avoid redundant gradient reduction in pipeline parallelism (multiple microbatch values should be reduced once),

colossalai/pipeline/schedule/one_f_one_b.py

@@ -7,7 +7,7 @@
 from torch.utils._pytree import tree_map
 
 from colossalai.accelerator import get_accelerator
-from colossalai.interface import ModelWrapper, OptimizerWrapper
+from colossalai.interface import OptimizerWrapper
 from colossalai.pipeline.p2p import PipelineP2PCommunication, create_send_metadata
 from colossalai.pipeline.stage_manager import PipelineStageManager
 from colossalai.utils import get_current_device
@@ -327,9 +327,7 @@ def run_forward_only(
             self.send_forward(output_obj)
 
         if outputs is not None:
-            if isinstance(model, ModelWrapper):
-                model = model.unwrap()
-            outputs = merge_batch(outputs, getattr(model, "batch_size_dim", 0))
+            outputs = merge_batch(outputs)
         return {"loss": accum_loss, "outputs": outputs}
 
     def run_forward_backward(
@@ -412,9 +410,7 @@ def run_forward_backward(
         assert all(len(v) == 0 for v in input_objs) and all(len(v) == 0 for v in output_objs)
 
         if outputs is not None:
-            if isinstance(model, ModelWrapper):
-                model = model.unwrap()
-            outputs = merge_batch(outputs, getattr(model, "batch_size_dim", 0))
+            outputs = merge_batch(outputs)
         return {"loss": accum_loss, "outputs": outputs}
 
     def forward_backward_step(

docs/source/en/advanced_tutorials/train_gpt_using_hybrid_parallelism.md

@@ -178,7 +178,7 @@ def train_epoch(
         for _ in pbar:
             if use_pipeline:
                 outputs = booster.execute_pipeline(
-                    train_dataloader_iter, model, _criterion, optimizer, return_loss=True, return_outputs=True
+                    train_dataloader_iter, model, _criterion, optimizer, return_loss=True
                 )
                 # Backward and optimize
                 if is_pp_last_stage:

docs/source/en/advanced_tutorials/train_vit_with_hybrid_parallelism.md

@@ -231,7 +231,7 @@ def run_forward_backward(
     if isinstance(booster.plugin, HybridParallelPlugin) and booster.plugin.pp_size > 1:
         # run pipeline forward backward when enabling pp in hybrid parallel plugin
         output_dict = booster.execute_pipeline(
-            data_iter, model, criterion, optimizer, return_loss=True, return_outputs=True
+            data_iter, model, criterion, optimizer, return_loss=True
         )
         loss, outputs = output_dict["loss"], output_dict["outputs"]
     else:

docs/source/en/features/pipeline_parallel.md

@@ -198,8 +198,7 @@ def train_epoch(epoch: int, model: nn.Module, optimizer: Optimizer, _criterion:
                                                 model,
                                                 _criterion,
                                                 optimizer,
-                                                return_loss=True,
-                                                return_outputs=True)
+                                                return_loss=True)
             # Backward and optimize
             if is_pp_last_stage:
                 loss = outputs['loss']

docs/source/en/features/shardformer.md

@@ -271,7 +271,7 @@ However, if pipeline parallel is enabled, there are several usages different fro
 3. Do forward and backward passing through calling `Booster.execute_pipeline` method:
     ```python
     outputs = booster.execute_pipeline(
-        train_dataloader_iter, model, _criterion, optimizer, return_loss=True, return_outputs=True
+        train_dataloader_iter, model, _criterion, optimizer, return_loss=True
     )
     ```
     Backward passing has been completed by this method, so there is no need to call `loss.backward()` after executing this method.

docs/source/zh-Hans/advanced_tutorials/train_gpt_using_hybrid_parallelism.md

@@ -175,7 +175,7 @@ def train_epoch(
         for _ in pbar:
             if use_pipeline:
                 outputs = booster.execute_pipeline(
-                    train_dataloader_iter, model, _criterion, optimizer, return_loss=True, return_outputs=True
+                    train_dataloader_iter, model, _criterion, optimizer, return_loss=True
                 )
                 # Backward and optimize
                 if is_pp_last_stage:

docs/source/zh-Hans/advanced_tutorials/train_vit_with_hybrid_parallelism.md

@@ -234,7 +234,7 @@ def run_forward_backward(
     if isinstance(booster.plugin, HybridParallelPlugin) and booster.plugin.pp_size > 1:
         # run pipeline forward backward when enabling pp in hybrid parallel plugin
         output_dict = booster.execute_pipeline(
-            data_iter, model, criterion, optimizer, return_loss=True, return_outputs=True
+            data_iter, model, criterion, optimizer, return_loss=True
         )
         loss, outputs = output_dict["loss"], output_dict["outputs"]
     else:

docs/source/zh-Hans/features/pipeline_parallel.md

@@ -193,8 +193,7 @@ def train_epoch(epoch: int, model: nn.Module, optimizer: Optimizer, _criterion:
                                                 model,
                                                 _criterion,
                                                 optimizer,
-                                                return_loss=True,
-                                                return_outputs=True)
+                                                return_loss=True)
             # Backward and optimize
             if is_pp_last_stage:
                 loss = outputs['loss']