remove SharedDDP as it is deprecated (huggingface#25702)

* remove SharedDDP as it was drepracated

* apply review suggestion

* make style

* Oops,forgot to remove the compute_loss context manager in Seq2SeqTrainer.

* remove the unnecessary conditional statement

* keep the logic of IPEX

* clean code

* mix precision setup & make fixup

---------

Co-authored-by: statelesshz <jihuazhong1@huawei.com>

examples/legacy/seq2seq/seq2seq_trainer.py

@@ -19,7 +19,6 @@
 from torch.utils.data import DistributedSampler, RandomSampler
 
 from transformers import PreTrainedModel, Trainer, logging
-from transformers.integrations import is_fairscale_available
 from transformers.models.fsmt.configuration_fsmt import FSMTConfig
 from transformers.optimization import (
     Adafactor,
@@ -36,10 +35,6 @@
 from transformers.utils import is_torch_tpu_available
 
 
-if is_fairscale_available():
-    from fairscale.optim import OSS
-
-
 logger = logging.get_logger(__name__)
 
 arg_to_scheduler = {
@@ -118,14 +113,7 @@ def create_optimizer_and_scheduler(self, num_training_steps: int):
                     "eps": self.args.adam_epsilon,
                 }
             optimizer_kwargs["lr"] = self.args.learning_rate
-            if self.sharded_ddp:
-                self.optimizer = OSS(
-                    params=optimizer_grouped_parameters,
-                    optim=optimizer_cls,
-                    **optimizer_kwargs,
-                )
-            else:
-                self.optimizer = optimizer_cls(optimizer_grouped_parameters, **optimizer_kwargs)
+            self.optimizer = optimizer_cls(optimizer_grouped_parameters, **optimizer_kwargs)
 
         if self.lr_scheduler is None:
             self.lr_scheduler = self._get_lr_scheduler(num_training_steps)

setup.py

@@ -109,7 +109,6 @@
     "diffusers",
     "dill<0.3.5",
     "evaluate>=0.2.0",
-    "fairscale>0.3",
     "faiss-cpu",
     "fastapi",
     "filelock",
@@ -275,7 +274,6 @@ def run(self):
 
 extras["sagemaker"] = deps_list("sagemaker")
 extras["deepspeed"] = deps_list("deepspeed") + extras["accelerate"]
-extras["fairscale"] = deps_list("fairscale")
 extras["optuna"] = deps_list("optuna")
 extras["ray"] = deps_list("ray[tune]")
 extras["sigopt"] = deps_list("sigopt")

src/transformers/dependency_versions_table.py

@@ -16,7 +16,6 @@
     "diffusers": "diffusers",
     "dill": "dill<0.3.5",
     "evaluate": "evaluate>=0.2.0",
-    "fairscale": "fairscale>0.3",
     "faiss-cpu": "faiss-cpu",
     "fastapi": "fastapi",
     "filelock": "filelock",

src/transformers/integrations/__init__.py

@@ -57,7 +57,6 @@
         "is_codecarbon_available",
         "is_comet_available",
         "is_dagshub_available",
-        "is_fairscale_available",
         "is_flyte_deck_standard_available",
         "is_flytekit_available",
         "is_mlflow_available",
@@ -118,7 +117,6 @@
         is_codecarbon_available,
         is_comet_available,
         is_dagshub_available,
-        is_fairscale_available,
         is_flyte_deck_standard_available,
         is_flytekit_available,
         is_mlflow_available,

src/transformers/integrations/integration_utils.py

@@ -134,10 +134,6 @@ def is_dagshub_available():
     return None not in [importlib.util.find_spec("dagshub"), importlib.util.find_spec("mlflow")]
 
 
-def is_fairscale_available():
-    return importlib.util.find_spec("fairscale") is not None
-
-
 def is_neptune_available():
     return _has_neptune
 

src/transformers/testing_utils.py

@@ -42,7 +42,6 @@
 
 from .integrations import (
     is_clearml_available,
-    is_fairscale_available,
     is_optuna_available,
     is_ray_available,
     is_sigopt_available,
@@ -871,13 +870,6 @@ def require_deepspeed(test_case):
     return unittest.skipUnless(is_deepspeed_available(), "test requires deepspeed")(test_case)
 
 
-def require_fairscale(test_case):
-    """
-    Decorator marking a test that requires fairscale
-    """
-    return unittest.skipUnless(is_fairscale_available(), "test requires fairscale")(test_case)
-
-
 def require_apex(test_case):
     """
     Decorator marking a test that requires apex