Deprecate PPOTrainer

examples/scripts/ppo/ppo.py

@@ -9,8 +9,7 @@
     HfArgumentParser,
 )
 
-from trl import ModelConfig
-from trl.trainer.ppov2_trainer import PPOv2Config, PPOv2Trainer
+from trl import ModelConfig, PPOv2Config, PPOv2Trainer
 from trl.trainer.utils import SIMPLE_QUERY_CHAT_TEMPLATE
 
 

examples/scripts/ppo/ppo_tldr.py

@@ -9,8 +9,7 @@
     HfArgumentParser,
 )
 
-from trl import ModelConfig
-from trl.trainer.ppov2_trainer import PPOv2Config, PPOv2Trainer
+from trl import ModelConfig, PPOv2Config, PPOv2Trainer
 from trl.trainer.utils import SIMPLE_QUERY_CHAT_TEMPLATE
 
 

trl/__init__.py

@@ -56,6 +56,8 @@
         "ORPOTrainer",
         "PPOConfig",
         "PPOTrainer",
+        "PPOv2Config",
+        "PPOv2Trainer",
         "RewardConfig",
         "RewardTrainer",
         "SFTConfig",
@@ -144,6 +146,8 @@
         ORPOTrainer,
         PPOConfig,
         PPOTrainer,
+        PPOv2Config,
+        PPOv2Trainer,
         RewardConfig,
         RewardTrainer,
         SFTConfig,

trl/trainer/__init__.py

@@ -47,6 +47,8 @@
     "orpo_trainer": ["ORPOTrainer"],
     "ppo_config": ["PPOConfig"],
     "ppo_trainer": ["PPOTrainer"],
+    "ppov2_config": ["PPOv2Config"],
+    "ppov2_trainer": ["PPOv2Trainer"],
     "reward_config": ["RewardConfig"],
     "reward_trainer": ["RewardTrainer", "compute_accuracy"],
     "sft_config": ["SFTConfig"],
@@ -112,6 +114,8 @@
     from .orpo_trainer import ORPOTrainer
     from .ppo_config import PPOConfig
     from .ppo_trainer import PPOTrainer
+    from .ppov2_config import PPOv2Config
+    from .ppov2_trainer import PPOv2Trainer
     from .reward_config import RewardConfig
     from .reward_trainer import RewardTrainer, compute_accuracy
     from .sft_config import SFTConfig

trl/trainer/ppo_config.py

@@ -204,6 +204,10 @@ class PPOConfig:
         optimize_device_cache = optimize_cuda_cache
 
     def __post_init__(self):
+        warnings.warn(
+            "`PPOConfig` is deprecated and will be removed in trl v0.12. Please use `PPOv2Config` with `PPOv2Trainer` instead.",
+            FutureWarning,
+        )
         if self.forward_batch_size is not None:
             warnings.warn(
                 "Note that using `forward_batch_size` is deprecated, use `mini_batch_size` instead. By setting it you overwrite `mini_batch_size` which affects both the batch size during forward passes and also the mini batch size for PPO optimization."

trl/trainer/ppo_trainer.py

@@ -188,6 +188,10 @@ def __init__(
             training_data_collator (Optional[function]):
                 Custom data collator used for training.
         """
+        warnings.warn(
+            "`PPOTrainer` is deprecated and will be removed in trl v0.12. Please use `PPOv2Trainer` instead.",
+            FutureWarning,
+        )
         super().__init__(config)
 
         # initial seed for reproducible experiments