Skip to content

Commit

Permalink
Remove deprecated evalutate_during_training (huggingface#8852)
Browse files Browse the repository at this point in the history
* Remove deprecated `evalutate_during_training`

* Update src/transformers/training_args_tf.py

Co-authored-by: Lysandre Debut <lysandre@huggingface.co>

Co-authored-by: Lysandre Debut <lysandre@huggingface.co>
  • Loading branch information
sgugger and LysandreJik authored Nov 30, 2020
1 parent 7738494 commit 5530299
Show file tree
Hide file tree
Showing 9 changed files with 23 additions and 12 deletions.
3 changes: 2 additions & 1 deletion examples/seq2seq/builtin_trainer/finetune.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@
python finetune_trainer.py \
--learning_rate=3e-5 \
--fp16 \
--do_train --do_eval --do_predict --evaluate_during_training \
--do_train --do_eval --do_predict \
--evaluation_strategy steps \
--predict_with_generate \
--n_val 1000 \
"$@"
3 changes: 2 additions & 1 deletion examples/seq2seq/builtin_trainer/finetune_tpu.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ export TPU_NUM_CORES=8
python xla_spawn.py --num_cores $TPU_NUM_CORES \
finetune_trainer.py \
--learning_rate=3e-5 \
--do_train --do_eval --evaluate_during_training \
--do_train --do_eval \
--evaluation_strategy steps \
--prediction_loss_only \
--n_val 1000 \
"$@"
3 changes: 2 additions & 1 deletion examples/seq2seq/builtin_trainer/train_distil_marian_enro.sh
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@ python finetune_trainer.py \
--num_train_epochs=6 \
--save_steps 3000 --eval_steps 3000 \
--max_source_length $MAX_LEN --max_target_length $MAX_LEN --val_max_target_length $MAX_LEN --test_max_target_length $MAX_LEN \
--do_train --do_eval --do_predict --evaluate_during_training\
--do_train --do_eval --do_predict \
--evaluation_strategy steps \
--predict_with_generate --logging_first_step \
--task translation --label_smoothing 0.1 \
"$@"
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@ python xla_spawn.py --num_cores $TPU_NUM_CORES \
--save_steps 500 --eval_steps 500 \
--logging_first_step --logging_steps 200 \
--max_source_length $MAX_LEN --max_target_length $MAX_LEN --val_max_target_length $MAX_LEN --test_max_target_length $MAX_LEN \
--do_train --do_eval --evaluate_during_training \
--do_train --do_eval \
--evaluation_strategy steps \
--prediction_loss_only \
--task translation --label_smoothing 0.1 \
"$@"
3 changes: 2 additions & 1 deletion examples/seq2seq/builtin_trainer/train_distilbart_cnn.sh
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ python finetune_trainer.py \
--save_steps 3000 --eval_steps 3000 \
--logging_first_step \
--max_target_length 56 --val_max_target_length $MAX_TGT_LEN --test_max_target_length $MAX_TGT_LEN \
--do_train --do_eval --do_predict --evaluate_during_training \
--do_train --do_eval --do_predict \
--evaluation_strategy steps \
--predict_with_generate --sortish_sampler \
"$@"
3 changes: 2 additions & 1 deletion examples/seq2seq/builtin_trainer/train_mbart_cc25_enro.sh
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@ python finetune_trainer.py \
--sortish_sampler \
--num_train_epochs 6 \
--save_steps 25000 --eval_steps 25000 --logging_steps 1000 \
--do_train --do_eval --do_predict --evaluate_during_training \
--do_train --do_eval --do_predict \
--evaluation_strategy steps \
--predict_with_generate --logging_first_step \
--task translation \
"$@"
5 changes: 3 additions & 2 deletions src/transformers/integrations.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import math
import os

from .trainer_utils import EvaluationStrategy
from .utils import logging


Expand Down Expand Up @@ -212,13 +213,13 @@ def _objective(trial, checkpoint_dir=None):
# Check for `do_eval` and `eval_during_training` for schedulers that require intermediate reporting.
if isinstance(
kwargs["scheduler"], (ASHAScheduler, MedianStoppingRule, HyperBandForBOHB, PopulationBasedTraining)
) and (not trainer.args.do_eval or not trainer.args.evaluate_during_training):
) and (not trainer.args.do_eval or trainer.args.evaluation_strategy == EvaluationStrategy.NO):
raise RuntimeError(
"You are using {cls} as a scheduler but you haven't enabled evaluation during training. "
"This means your trials will not report intermediate results to Ray Tune, and "
"can thus not be stopped early or used to exploit other trials parameters. "
"If this is what you want, do not use {cls}. If you would like to use {cls}, "
"make sure you pass `do_eval=True` and `evaluate_during_training=True` in the "
"make sure you pass `do_eval=True` and `evaluation_strategy='steps'` in the "
"Trainer `args`.".format(cls=type(kwargs["scheduler"]).__name__)
)

Expand Down
4 changes: 2 additions & 2 deletions src/transformers/trainer_tf.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

from .modeling_tf_utils import TFPreTrainedModel
from .optimization_tf import GradientAccumulator, create_optimizer
from .trainer_utils import PREFIX_CHECKPOINT_DIR, EvalPrediction, PredictionOutput, set_seed
from .trainer_utils import PREFIX_CHECKPOINT_DIR, EvalPrediction, EvaluationStrategy, PredictionOutput, set_seed
from .training_args_tf import TFTrainingArguments
from .utils import logging

Expand Down Expand Up @@ -561,7 +561,7 @@ def train(self) -> None:

if (
self.args.eval_steps > 0
and self.args.evaluate_during_training
and self.args.evaluate_strategy == EvaluationStrategy.STEPS
and self.global_step % self.args.eval_steps == 0
):
self.evaluate()
Expand Down
8 changes: 6 additions & 2 deletions src/transformers/training_args_tf.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,12 @@ class TFTrainingArguments(TrainingArguments):
Whether to run evaluation on the dev set or not.
do_predict (:obj:`bool`, `optional`, defaults to :obj:`False`):
Whether to run predictions on the test set or not.
evaluate_during_training (:obj:`bool`, `optional`, defaults to :obj:`False`):
Whether to run evaluation during training at each logging step or not.
evaluation_strategy (:obj:`str` or :class:`~transformers.trainer_utils.EvaluationStrategy`, `optional`, defaults to :obj:`"no"`):
The evaluation strategy to adopt during training. Possible values are:
* :obj:`"no"`: No evaluation is done during training.
* :obj:`"steps"`: Evaluation is done (and logged) every :obj:`eval_steps`.
per_device_train_batch_size (:obj:`int`, `optional`, defaults to 8):
The batch size per GPU/TPU core/CPU for training.
per_device_eval_batch_size (:obj:`int`, `optional`, defaults to 8):
Expand Down

0 comments on commit 5530299

Please sign in to comment.