Adding Early stopping feature in ImageClassification (WIP)

docs/samples/Microsoft.ML.Samples/Dynamic/ImageClassification/ResnetV2101TransferLearningTrainTestSplit.cs

@@ -67,6 +67,7 @@ public static void Example()
                     epoch: 50,
                     batchSize: 10,
                     learningRate: 0.01f,
+                    enableEarlyStopping: true,
                     metricsCallback: (metrics) => Console.WriteLine(metrics),
                     validationSet: testDataset);
 

src/Microsoft.ML.Dnn/DnnCatalog.cs

@@ -89,6 +89,9 @@ public static DnnRetrainEstimator RetrainDnnModel(
         /// <param name="epoch">Number of training iterations. Each iteration/epoch refers to one pass over the dataset.</param>
         /// <param name="batchSize">The batch size for training.</param>
         /// <param name="learningRate">The learning rate for training.</param>
+        /// <param name="enableEarlyStopping">Whether early stopping technique should be used when accuracy stops improving.</param>
+        /// <param name="earlyStoppingminDelta">Minimum change in accuracy to qualify as improvement.</param>
+        /// <param name="earlyStoppingPatience">Number of epochs to wait after no improvement is observed before early stopping.</param>
         /// <param name="metricsCallback">Callback for reporting model statistics during training phase.</param>
         /// <param name="statisticFrequency">Indicates the frequency of epochs at which to report model statistics during training phase.</param>
         /// <param name="framework">Indicates the choice of DNN training framework. Currently only tensorflow is supported.</param>
@@ -113,6 +116,9 @@ public static ImageClassificationEstimator ImageClassification(
             int epoch = 100,
             int batchSize = 10,
             float learningRate = 0.01f,
+            bool enableEarlyStopping = true,
+            float earlyStoppingminDelta = 0.01f,
+            int earlyStoppingPatience = 20,
             ImageClassificationMetricsCallback metricsCallback = null,
             int statisticFrequency = 1,
             DnnFramework framework = DnnFramework.Tensorflow,
@@ -136,6 +142,9 @@ public static ImageClassificationEstimator ImageClassification(
                 Epoch = epoch,
                 LearningRate = learningRate,
                 BatchSize = batchSize,
+                EnableEarlyStopping = enableEarlyStopping,
+                MinDelta = earlyStoppingminDelta,
+                Patience = earlyStoppingPatience,
                 ScoreColumnName = scoreColumnName,
                 PredictedLabelColumnName = predictedLabelColumnName,
                 FinalModelPrefix = finalModelPrefix,

src/Microsoft.ML.Dnn/ImageClassificationTransform.cs

@@ -338,7 +338,13 @@ private void TrainAndEvaluateClassificationLayer(string trainBottleneckFilePath,
 
             ImageClassificationMetrics metrics = new ImageClassificationMetrics();
             metrics.Train = new TrainMetrics();
-            for (int epoch = 0; epoch < epochs; epoch += 1)
+            //Early stopping variables
+            bool earlyStop = false;
+            int wait = 0;
+            var history = new TrainMetrics();
+            history.Accuracy = 0;
+
+            for (int epoch = 0; epoch < epochs & !earlyStop; epoch += 1)
             {
                 metrics.Train.Accuracy = 0;
                 metrics.Train.CrossEntropy = 0;
@@ -445,6 +451,24 @@ private void TrainAndEvaluateClassificationLayer(string trainBottleneckFilePath,
                         statisticsCallback(metrics);
                     }
                 }
+                // early stopping check
+                if (options.EnableEarlyStopping)
+                {
+                    if (metrics.Train.Accuracy - options.MinDelta > history.Accuracy)
+                    {
+                        history.Accuracy = metrics.Train.Accuracy;
+                        wait = 0;
+                    }
+                    else
+                    {
+                        wait += 1;
+                        if (wait >= options.Patience)
+                        {
+                            Console.WriteLine("*** Early Stopping at epoch " + epoch.ToString());
+                            earlyStop = true;
+                        }
+                    }
+                }
             }
 
             trainSaver.save(_session, _checkpointPath);
@@ -1065,6 +1089,24 @@ internal sealed class Options : TransformInputBase
             [Argument(ArgumentType.AtMostOnce, HelpText = "Learning rate to use during optimization.", SortOrder = 12)]
             public float LearningRate = 0.01f;
 
+            /// <summary>
+            /// Whether early stopping technique should be used when accuracy stops improving.
+            /// </summary>
+            [Argument(ArgumentType.AtMostOnce, HelpText = "Whether early stopping technique should be used when accuracy stops improving.", SortOrder = 15)]
+            public bool EnableEarlyStopping = true;
+
+            /// <summary>
+            /// Minimum change in accuracy to qualify as improvement.
+            /// </summary>
+            [Argument(ArgumentType.AtMostOnce, HelpText = "Minimum change in accuracy to qualify as improvement.", SortOrder = 15)]
+            public float MinDelta = 0.0f;
+
+            /// <summary>
+            /// Number of epochs to wait after no improvement is observed before early stopping.
+            /// </summary>
+            [Argument(ArgumentType.AtMostOnce, HelpText = "Number of epochs to wait after no improvement is observed before early stopping.", SortOrder = 15)]
+            public int Patience = 20;
+
             /// <summary>
             /// Specifies the model architecture to be used in the case of image classification training using transfer learning.
             /// </summary>