Added learning rate scheduling for cifar and options for using resnet_v2_50 model

docs/samples/Microsoft.ML.Samples/Dynamic/ImageClassification/ResnetV50TransferLearningTrainTestSplit.cs

@@ -0,0 +1,321 @@
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.IO.Compression;
+using System.Linq;
+using System.Net;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.ML;
+using Microsoft.ML.Data;
+using Microsoft.ML.Transforms;
+using static Microsoft.ML.DataOperationsCatalog;
+
+namespace Samples.Dynamic
+{
+    public class ResnetV50TransferLearningTrainTestSplit
+    {
+        public static void Example()
+        {
+            string assetsRelativePath = @"../../../assets";
+            string assetsPath = GetAbsolutePath(assetsRelativePath);
+
+            var outputMlNetModelFilePath = Path.Combine(assetsPath, "outputs",
+                "imageClassifier.zip");
+
+            string imagesDownloadFolderPath = Path.Combine(assetsPath, "inputs",
+                "images");
+
+            //This sample tests resnet_v2_50 with CIFAR-10 dataset
+            string finalImagesFolderNameTrain = "cifar\\train";
+            string fullImagesetFolderPathTrain = Path.Combine(
+                imagesDownloadFolderPath, finalImagesFolderNameTrain);
+
+            string finalImagesFolderNameTest = "cifar\\test";
+            string fullImagesetFolderPathTest = Path.Combine(
+                imagesDownloadFolderPath, finalImagesFolderNameTest);
+            try
+            {
+
+                MLContext mlContext = new MLContext(seed: 1);
+
+                //Load all the original train images info
+                IEnumerable<ImageData> train_images = LoadImagesFromDirectory(
+                    folder: fullImagesetFolderPathTrain, useFolderNameAsLabel: true);
+
+                IDataView trainDataset = mlContext.Data.LoadFromEnumerable(train_images);
+
+                trainDataset = mlContext.Transforms.Conversion
+                    .MapValueToKey(outputColumnName: "Label",
+                                   inputColumnName: "Label",
+                                   keyOrdinality: ValueToKeyMappingEstimator.KeyOrdinality.ByValue)
+                    .Fit(trainDataset)
+                    .Transform(trainDataset);
+
+                //Load all the original test images info
+                IEnumerable<ImageData> test_images = LoadImagesFromDirectory(
+                    folder: fullImagesetFolderPathTest, useFolderNameAsLabel: true);
+
+                IDataView testDataset = mlContext.Data.LoadFromEnumerable(test_images);
+
+                testDataset = mlContext.Transforms.Conversion
+                    .MapValueToKey(outputColumnName: "Label",
+                                   inputColumnName: "Label",
+                                   keyOrdinality: ValueToKeyMappingEstimator.KeyOrdinality.ByValue)
+                    .Fit(testDataset)
+                    .Transform(testDataset);
+
+                // Split the data 90:10 into train and test sets, train and evaluate.
+                var validationSet = mlContext.Transforms.LoadImages("Image", fullImagesetFolderPathTest, false, "ImagePath") // false indicates we want the image as a VBuffer<byte>
+                    .Fit(testDataset)
+                    .Transform(testDataset);
+
+                var pipeline = mlContext.Transforms.LoadImages("Image", fullImagesetFolderPathTrain, false, "ImagePath") // false indicates we want the image as a VBuffer<byte>
+                    .Append(mlContext.Model.ImageClassification(
+                        "Image", "Label",
+                        // Just by changing/selecting InceptionV3 here instead of 
+                        // ResnetV2101 you can try a different architecture/pre-trained 
+                        // model. 
+                        arch: ImageClassificationEstimator.Architecture.ResnetV250,
+                        epoch: 182,
+                        batchSize: 128,
+                        //learningRate: 0.01f,
+                        uselearningRateScheduling: true,
+                        metricsCallback: (metrics) => Console.WriteLine(metrics),
+                        validationSet: validationSet,
+                        reuseValidationSetBottleneckCachedValues: false,
+                        reuseTrainSetBottleneckCachedValues :false,
+                        disableEarlyStopping: true)
+                    .Append(mlContext.Transforms.Conversion.MapKeyToValue(outputColumnName: "PredictedLabel", inputColumnName: "PredictedLabel")));
+
+
+                Console.WriteLine("*** Training the image classification model with " +
+                    "DNN Transfer Learning on top of the selected pre-trained " +
+                    "model/architecture ***");
+
+                // Measuring training time
+                var watch = System.Diagnostics.Stopwatch.StartNew();
+
+                var trainedModel = pipeline.Fit(trainDataset);
+
+                watch.Stop();
+                long elapsedMs = watch.ElapsedMilliseconds;
+
+                Console.WriteLine("Training with transfer learning took: " +
+                    (elapsedMs / 1000).ToString() + " seconds");
+
+                mlContext.Model.Save(trainedModel, testDataset.Schema,
+                    "model.zip");
+
+                ITransformer loadedModel;
+                DataViewSchema schema;
+                using (var file = File.OpenRead("model.zip"))
+                    loadedModel = mlContext.Model.Load(file, out schema);
+
+                EvaluateModel(mlContext, testDataset, loadedModel);
+
+                watch = System.Diagnostics.Stopwatch.StartNew();
+
+                TrySinglePrediction(fullImagesetFolderPathTest, mlContext, loadedModel);
+
+                watch.Stop();
+                elapsedMs = watch.ElapsedMilliseconds;
+
+                Console.WriteLine("Prediction engine took: " +
+                    (elapsedMs / 1000).ToString() + " seconds");
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine(ex.ToString());
+            }
+
+            Console.WriteLine("Press any key to finish");
+            Console.ReadKey();
+        }
+
+        private static void TrySinglePrediction(string imagesForPredictions,
+            MLContext mlContext, ITransformer trainedModel)
+        {
+            // Create prediction function to try one prediction
+            var predictionEngine = mlContext.Model
+                .CreatePredictionEngine<ImageData, ImagePrediction>(trainedModel);
+
+            IEnumerable<ImageData> testImages = LoadImagesFromDirectory(
+                imagesForPredictions, false);
+
+            ImageData imageToPredict = new ImageData
+            {
+                ImagePath = testImages.First().ImagePath
+            };
+
+            var prediction = predictionEngine.Predict(imageToPredict);
+
+            Console.WriteLine($"ImageFile : " +
+                $"[{Path.GetFileName(imageToPredict.ImagePath)}], " +
+                $"Scores : [{string.Join(",", prediction.Score)}], " +
+                $"Predicted Label : {prediction.PredictedLabel}");
+        }
+
+
+        private static void EvaluateModel(MLContext mlContext,
+            IDataView testDataset, ITransformer trainedModel)
+        {
+            Console.WriteLine("Making bulk predictions and evaluating model's " +
+                "quality...");
+
+            // Measuring time
+            var watch2 = System.Diagnostics.Stopwatch.StartNew();
+
+            IDataView predictions = trainedModel.Transform(testDataset);
+            var metrics = mlContext.MulticlassClassification.Evaluate(predictions);
+
+            Console.WriteLine($"Micro-accuracy: {metrics.MicroAccuracy}," +
+                              $"macro-accuracy = {metrics.MacroAccuracy}");
+
+            watch2.Stop();
+            long elapsed2Ms = watch2.ElapsedMilliseconds;
+
+            Console.WriteLine("Predicting and Evaluation took: " +
+                (elapsed2Ms / 1000).ToString() + " seconds");
+        }
+
+        public static IEnumerable<ImageData> LoadImagesFromDirectory(string folder,
+            bool useFolderNameAsLabel = true)
+        {
+            var files = Directory.GetFiles(folder, "*",
+                searchOption: SearchOption.AllDirectories);
+            foreach (var file in files)
+            {
+                if (Path.GetExtension(file) != ".jpg" &&
+                    Path.GetExtension(file) != ".JPEG" &&
+                    Path.GetExtension(file) != ".png")
+                    continue;
+
+                var label = Path.GetFileName(file);
+                if (useFolderNameAsLabel)
+                    label = Directory.GetParent(file).Name;
+                else
+                {
+                    for (int index = 0; index < label.Length; index++)
+                    {
+                        if (!char.IsLetter(label[index]))
+                        {
+                            label = label.Substring(0, index);
+                            break;
+                        }
+                    }
+                }
+
+                yield return new ImageData()
+                {
+                    ImagePath = file,
+                    Label = label
+                };
+
+            }
+        }
+
+        public static string DownloadImageSet(string imagesDownloadFolder)
+        {
+            // get a set of images to teach the network about the new classes
+
+            //SINGLE SMALL FLOWERS IMAGESET (200 files)
+            string fileName = "flower_photos_small_set.zip";
+            string url = $"https://mlnetfilestorage.file.core.windows.net/" +
+                $"imagesets/flower_images/flower_photos_small_set.zip?st=2019-08-" +
+                $"07T21%3A27%3A44Z&se=2030-08-08T21%3A27%3A00Z&sp=rl&sv=2018-03-" +
+                $"28&sr=f&sig=SZ0UBX47pXD0F1rmrOM%2BfcwbPVob8hlgFtIlN89micM%3D";
+
+            Download(url, imagesDownloadFolder, fileName);
+            UnZip(Path.Combine(imagesDownloadFolder, fileName), imagesDownloadFolder);
+
+            return Path.GetFileNameWithoutExtension(fileName);
+        }
+
+        public static bool Download(string url, string destDir, string destFileName)
+        {
+            if (destFileName == null)
+                destFileName = url.Split(Path.DirectorySeparatorChar).Last();
+
+            Directory.CreateDirectory(destDir);
+
+            string relativeFilePath = Path.Combine(destDir, destFileName);
+
+            if (File.Exists(relativeFilePath))
+            {
+                Console.WriteLine($"{relativeFilePath} already exists.");
+                return false;
+            }
+
+            var wc = new WebClient();
+            Console.WriteLine($"Downloading {relativeFilePath}");
+            var download = Task.Run(() => wc.DownloadFile(url, relativeFilePath));
+            while (!download.IsCompleted)
+            {
+                Thread.Sleep(1000);
+                Console.Write(".");
+            }
+            Console.WriteLine("");
+            Console.WriteLine($"Downloaded {relativeFilePath}");
+
+            return true;
+        }
+
+        public static void UnZip(String gzArchiveName, String destFolder)
+        {
+            var flag = gzArchiveName.Split(Path.DirectorySeparatorChar)
+                .Last()
+                .Split('.')
+                .First() + ".bin";
+
+            if (File.Exists(Path.Combine(destFolder, flag))) return;
+
+            Console.WriteLine($"Extracting.");
+            var task = Task.Run(() =>
+            {
+                ZipFile.ExtractToDirectory(gzArchiveName, destFolder);
+            });
+
+            while (!task.IsCompleted)
+            {
+                Thread.Sleep(200);
+                Console.Write(".");
+            }
+
+            File.Create(Path.Combine(destFolder, flag));
+            Console.WriteLine("");
+            Console.WriteLine("Extracting is completed.");
+        }
+
+        public static string GetAbsolutePath(string relativePath)
+        {
+            FileInfo _dataRoot = new FileInfo(typeof(
+                ResnetV2101TransferLearningTrainTestSplit).Assembly.Location);
+
+            string assemblyFolderPath = _dataRoot.Directory.FullName;
+
+            string fullPath = Path.Combine(assemblyFolderPath, relativePath);
+
+            return fullPath;
+        }
+
+        public class ImageData
+        {
+            [LoadColumn(0)]
+            public string ImagePath;
+
+            [LoadColumn(1)]
+            public string Label;
+        }
+
+        public class ImagePrediction
+        {
+            [ColumnName("Score")]
+            public float[] Score;
+
+            [ColumnName("PredictedLabel")]
+            public string PredictedLabel;
+        }
+    }
+}

src/Microsoft.ML.Dnn/DnnCatalog.cs

@@ -101,6 +101,7 @@ public static DnnRetrainEstimator RetrainDnnModel(
         /// <param name="epoch">Number of training iterations. Each iteration/epoch refers to one pass over the dataset.</param>
         /// <param name="batchSize">The batch size for training.</param>
         /// <param name="learningRate">The learning rate for training.</param>
+        /// <param name="uselearningRateScheduling">Enables learning rate scheduling to use during optimization if true. Only use for Cifar dataset.</param>
         /// <param name="disableEarlyStopping">Whether to disable use of early stopping technique. Training will go on for the full epoch count.</param>
         /// <param name="earlyStopping">Early stopping technique parameters to be used to terminate training when training metric stops improving.</param>
         /// <param name="metricsCallback">Callback for reporting model statistics during training phase.</param>
@@ -127,6 +128,7 @@ public static ImageClassificationEstimator ImageClassification(
             int epoch = 100,
             int batchSize = 10,
             float learningRate = 0.01f,
+            bool uselearningRateScheduling = false,
             bool disableEarlyStopping = false,
             EarlyStopping earlyStopping = null,
             ImageClassificationMetricsCallback metricsCallback = null,
@@ -144,13 +146,14 @@ public static ImageClassificationEstimator ImageClassification(
         {
             var options = new ImageClassificationEstimator.Options()
             {
-                ModelLocation = arch == Architecture.ResnetV2101 ? @"resnet_v2_101_299.meta" : @"InceptionV3.meta",
+                ModelLocation = arch == Architecture.ResnetV2101 ?  @"resnet_v2_101_299.meta" : (arch == Architecture.ResnetV250 ? @"resnet_v2_50_299.meta" : @"InceptionV3.meta"),
                 InputColumns = new[] { featuresColumnName },
                 OutputColumns = new[] { scoreColumnName, predictedLabelColumnName },
                 LabelColumn = labelColumnName,
                 TensorFlowLabel = labelColumnName,
                 Epoch = epoch,
                 LearningRate = learningRate,
+                UseLearningRateScheduling = uselearningRateScheduling,
                 BatchSize = batchSize,
                 EarlyStoppingCriteria = disableEarlyStopping ? null : earlyStopping == null ? new EarlyStopping() : earlyStopping,
                 ScoreColumnName = scoreColumnName,

src/Microsoft.ML.Dnn/GradientDescentOptimizerTensor.cs

@@ -0,0 +1,41 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Tensorflow;
+using Tensorflow.Framework;
+using static Tensorflow.Binding;
+
+namespace Microsoft.ML.Transforms
+{
+    /// <summary>
+    /// Optimizer that implements the gradient descent algorithm.
+    /// </summary>
+    public class GradientDescentOptimizerTensor : Optimizer
+    {
+        /// <summary>
+        /// Construct a new gradient descent optimizer.
+        /// </summary>
+        /// <param name="learningRateTensor">A Tensor or a floating point value.  The learning
+        /// rate to use.</param>
+        /// <param name="useLocking">If true use locks for update operations.</param>
+        /// <param name="name">Optional name prefix for the operations created when applying
+        /// gradients.Defaults to "GradientDescent".</param>
+        /// <remarks>
+        /// When eager execution is enabled, `learning_rate` can be a callable that
+        /// takes no arguments and returns the actual value to use.This can be useful
+        /// for changing these values across different invocations of optimizer
+        /// functions.
+        /// </remarks>
+        public GradientDescentOptimizerTensor(Tensor learningRateTensor, bool useLocking = false, string name = "GradientDescent")
+            : base(learningRateTensor, useLocking, name)
+        {
+            _lr_t = learningRateTensor;
+        }
+
+        public override void _prepare()
+        {
+            //var lr = _call_if_callable(_lr);
+            //_lr_t = ops.convert_to_tensor(lr, name: "learning_rate");
+        }
+    }
+}