【AutoParallel】Change llama in auto-parallel (#8151)

* change llama in auto * change ci * polish
PaddlePaddle · Mar 26, 2024 · 4d49a3e · 4d49a3e
1 parent 89bff20
commit 4d49a3e
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 9 deletions.
diff --git a/paddlenlp/transformers/llama/modeling_auto.py b/paddlenlp/transformers/llama/modeling_auto.py
@@ -1071,14 +1071,6 @@ def forward(self, prediction_scores, masked_lm_labels):
                 )
                 self.loss_func = paddle.nn.CrossEntropyLoss(reduction="none", ignore_index=self.ignore_index)
 
-        # Force Replicated to match dy & st
-        prediction_scores = dist.reshard(
-            prediction_scores,
-            get_mesh(-1),
-            [dist.Replicate(), dist.Replicate(), dist.Replicate()],
-        )
-        masked_lm_labels = dist.reshard(masked_lm_labels, get_mesh(-1), [dist.Replicate(), dist.Replicate()])
-
         # Force entropy same kernel
         with paddle.amp.auto_cast(False):
             if isinstance(prediction_scores, paddle.Tensor):

diff --git a/scripts/distribute/ci_case_auto.sh b/scripts/distribute/ci_case_auto.sh
@@ -1503,7 +1503,7 @@ function llama_dygraph_auto_bs8_fp16_DP2-MP2-PP2() {
     ips=-1
     mem=-1
     echo "result: loss=$loss ips=$ips mem=$mem"
-    loss_base=9.38257694
+    loss_base=9.38256836
     ips_base=-1
     mem_base=-1
     check_result $FUNCNAME ${loss_base} ${loss} ${ips_base} ${ips} ${mem_base} ${mem}

diff --git a/scripts/distribute/run_ci.sh b/scripts/distribute/run_ci.sh
@@ -27,7 +27,9 @@ target_lists_for_gpt=(
 
 target_lists_for_llama=(
     "llm/llama/auto_parallel"
+    "paddlenlp/trainer/auto_trainer.py"
     "paddlenlp/transformers/llama/modeling_auto_static.py"
+    "paddlenlp/transformers/llama/modeling_auto.py"
     "scripts/distribute"
 )