Integrate Phi3 LCE support

linkedin/Liger-Kernel#103
axolotl-ai-cloud · Aug 28, 2024 · 5bc5d51 · 5bc5d51
1 parent 89e0c2c
commit 5bc5d51
Showing 1 changed file with 4 additions and 3 deletions.
diff --git a/src/axolotl/integrations/liger/__init__.py b/src/axolotl/integrations/liger/__init__.py
@@ -167,6 +167,9 @@ def pre_model_load(self, cfg):
                 )
 
         elif cfg.model_config_type == "phi3":
+            from liger_kernel.transformers.model.phi3 import (
+                lce_forward as phi3_lce_forward,
+            )
             from transformers.models.phi3 import modeling_phi3
 
             if cfg.liger_rope:
@@ -178,6 +181,4 @@ def pre_model_load(self, cfg):
             if cfg.liger_cross_entropy:
                 modeling_phi3.CrossEntropyLoss = LigerCrossEntropyLoss
             if cfg.liger_fused_linear_cross_entropy:
-                logging.warning(
-                    "Fused linear cross entropy is not supported for Phi 3."
-                )
+                modeling_phi3.Phi3ForCausalLM.forward = phi3_lce_forward