Image classification using DNNs and Transfer Learning.

docs/samples/Microsoft.ML.Samples/Dynamic/ImageClassification/InceptionV3TransferLearning.cs

@@ -0,0 +1,109 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using Microsoft.ML;
+using Microsoft.ML.Data;
+using Microsoft.ML.Transforms;
+
+namespace Samples.Dynamic
+{
+    public static class InceptionV3TransferLearning
+    {
+        /// <summary>
+        /// Example use of Image classification API in a ML.NET pipeline.
+        /// </summary>
+        public static void Example()
+        {
+            var mlContext = new MLContext(seed: 1);
+
+            var imagesDataFile = Path.GetDirectoryName(
+                Microsoft.ML.SamplesUtils.DatasetUtils.DownloadImages());
+
+            var data = mlContext.Data.LoadFromEnumerable(
+                ImageNetData.LoadImagesFromDirectory(imagesDataFile, 4));
+
+            data = mlContext.Data.ShuffleRows(data, 5);
+            var pipeline = mlContext.Transforms.Conversion.MapValueToKey("Label")
+                .Append(mlContext.Transforms.LoadImages("ImageObject", null,
+                    "ImagePath"))
+                .Append(mlContext.Transforms.ResizeImages("Image",
+                    inputColumnName: "ImageObject", imageWidth: 299,
+                    imageHeight: 299))
+                .Append(mlContext.Transforms.ExtractPixels("Image",
+                    interleavePixelColors: true))
+                .Append(mlContext.Model.ImageClassification("Image",
+                    "Label", arch: DnnEstimator.Architecture.InceptionV3, epoch: 4,
+                    batchSize: 4));
+
+            var trainedModel = pipeline.Fit(data);
+            var predicted = trainedModel.Transform(data);
+            var metrics = mlContext.MulticlassClassification.Evaluate(predicted);
+
+            Console.WriteLine($"Micro-accuracy: {metrics.MicroAccuracy}," +
+                $"macro-accuracy = {metrics.MacroAccuracy}");
+
+            // Create prediction function and test prediction
+            var predictFunction = mlContext.Model
+                .CreatePredictionEngine<ImageNetData, ImagePrediction>(trainedModel);
+
+            var prediction = predictFunction
+                .Predict(ImageNetData.LoadImagesFromDirectory(imagesDataFile, 4)
+                .First());
+
+            Console.WriteLine($"Scores : [{string.Join(",", prediction.Score)}], " +
+                $"Predicted Label : {prediction.PredictedLabel}");
+
+        }
+    }
+
+    public class ImageNetData
+    {
+        [LoadColumn(0)]
+        public string ImagePath;
+
+        [LoadColumn(1)]
+        public string Label;
+
+        public static IEnumerable<ImageNetData> LoadImagesFromDirectory(
+            string folder, int repeat = 1, bool useFolderNameasLabel = false)
+        {
+            var files = Directory.GetFiles(folder, "*",
+                searchOption: SearchOption.AllDirectories);
+
+            foreach (var file in files)
+            {
+                if (Path.GetExtension(file) != ".jpg")
+                    continue;
+
+                var label = Path.GetFileName(file);
+                if (useFolderNameasLabel)
+                    label = Directory.GetParent(file).Name;
+                else
+                {
+                    for (int index = 0; index < label.Length; index++)
+                    {
+                        if (!char.IsLetter(label[index]))
+                        {
+                            label = label.Substring(0, index);
+                            break;
+                        }
+                    }
+                }
+
+                for (int index = 0; index < repeat; index++)
+                    yield return new ImageNetData() {
+                        ImagePath = file,Label = label };
+            }
+        }
+    }
+
+    public class ImagePrediction
+    {
+        [ColumnName("Score")]
+        public float[] Score;
+
+        [ColumnName("PredictedLabel")]
+        public Int64 PredictedLabel;
+    }
+}

docs/samples/Microsoft.ML.Samples/Dynamic/ImageClassification/ResnetV2101TransferLearning.cs

@@ -0,0 +1,108 @@
+using System;
+using System.Diagnostics;
+using System.Linq;
+using Microsoft.ML;
+using Microsoft.ML.Data;
+
+namespace Samples.Dynamic
+{
+    public static class ResnetV2101TransferLearning
+    {
+        /// <summary>
+        /// Example use of Image classification API in a ML.NET pipeline.
+        /// </summary>
+        public static void Example()
+        {
+            var sw = new Stopwatch();
+
+            var mlContext = new MLContext();
+            var data = GetTensorData();
+            var idv = mlContext.Data.LoadFromEnumerable(data);
+
+            // Create a ML pipeline.
+            var pipeline =
+                mlContext.Transforms.Conversion.MapValueToKey(
+                    nameof(TensorData.Label))
+                .Append(mlContext.Model.ImageClassification(
+                    nameof(TensorData.input),
+                    nameof(TensorData.Label), batchSize: 2));
+
+            // Run the pipeline and get the transformed values.
+            var estimator = pipeline.Fit(idv);
+            var transformedValues = estimator.Transform(idv);
+
+            // Retrieve model scores.
+            var outScores = mlContext.Data.CreateEnumerable<OutputScores>(
+                transformedValues, reuseRowObject: false);
+
+            // Display scores. (for the sake of brevity we display scores of the
+            // first 3 classes)
+            foreach (var prediction in outScores)
+            {
+                int numClasses = 0;
+                foreach (var classScore in prediction.Scores.Take(2))
+                {
+                    Console.WriteLine(
+                        $"Class #{numClasses++} score = {classScore}");
+                }
+                Console.WriteLine(
+                    $"Predicted Label: {prediction.PredictedLabel}");
+
+
+                Console.WriteLine(new string('-', 10));
+            }
+
+            Console.WriteLine(sw.Elapsed);
+            // Class #0 score = 0.03416626
+            // Class #1 score = 0.9658337
+            // Predicted Label: 1
+            // ----------
+            // Class #0 score = 0.02959618
+            // Class #1 score = 0.9704038
+            // Predicted Label: 1
+            // ----------
+        }
+
+        private const int imageHeight = 224; 
+        private const int imageWidth = 224;
+        private const int numChannels = 3;
+        private const int inputSize = imageHeight * imageWidth * numChannels;
+
+        /// <summary>
+        /// A class to hold sample tensor data. 
+        /// Member name should match the inputs that the model expects (in this
+        /// case, input).
+        /// </summary>
+        public class TensorData
+        {
+            [VectorType(imageHeight, imageWidth, numChannels)]
+            public float[] input { get; set; }
+
+            public Int64 Label { get; set; }
+        }
+
+        /// <summary>
+        /// Method to generate sample test data. Returns 2 sample rows.
+        /// </summary>
+        public static TensorData[] GetTensorData()
+        {
+            // This can be any numerical data. Assume image pixel values.
+            var image1 = Enumerable.Range(0, inputSize).Select(
+                x => (float)x / inputSize).ToArray();
+
+            var image2 = Enumerable.Range(0, inputSize).Select(
+                x => (float)(x + 10000) / inputSize).ToArray();
+            return new TensorData[] { new TensorData() { input = image1, Label = 0 },
+                new TensorData() { input = image2, Label = 1 } };
+        }
+
+        /// <summary>
+        /// Class to contain the output values from the transformation.
+        /// </summary>
+        class OutputScores
+        {
+            public float[] Scores { get; set; }
+            public Int64 PredictedLabel { get; set; }
+        }
+    }
+}

docs/samples/Microsoft.ML.Samples/Microsoft.ML.Samples.csproj

@@ -10,6 +10,7 @@
   </PropertyGroup>
 
   <ItemGroup>
+	<ProjectReference Include="..\..\..\src\Microsoft.ML.Dnn\Microsoft.ML.Dnn.csproj" />
     <ProjectReference Include="..\..\..\src\Microsoft.ML.LightGbm\Microsoft.ML.LightGbm.csproj" />
     <ProjectReference Include="..\..\..\src\Microsoft.ML.Mkl.Components\Microsoft.ML.Mkl.Components.csproj" />
     <ProjectReference Include="..\..\..\src\Microsoft.ML.KMeansClustering\Microsoft.ML.KMeansClustering.csproj" />
@@ -963,5 +964,10 @@
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
   </ItemGroup>
-
+    <ItemGroup>
+    <Content Include="$(ObjDir)DnnImageModels\Resnet101V2Tensorflow\resnet_v2_101_299.meta">
+      <Link>DnnImageModels\Resnet101V2Tensorflow\resnet_v2_101_299.meta</Link>
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+  </ItemGroup>
 </Project>

docs/samples/Microsoft.ML.Samples/Program.cs

@@ -10,7 +10,7 @@ public static class Program
 
         internal static void RunAll()
         {
-            int samples = 0;
+            /*int samples = 0;
             foreach (var type in Assembly.GetExecutingAssembly().GetTypes())
             {
                 var sample = type.GetMethod("Example", BindingFlags.Public | BindingFlags.Static | BindingFlags.FlattenHierarchy);
@@ -23,7 +23,8 @@ internal static void RunAll()
                 }
             }
 
-            Console.WriteLine("Number of samples that ran without any exception: " + samples);
+            Console.WriteLine("Number of samples that ran without any exception: " + samples);*/
+            InceptionV3TransferLearning.Example();
         }
     }
 }

pkg/Microsoft.ML.Dnn/Microsoft.ML.Dnn.nupkgproj

@@ -0,0 +1,12 @@
+<Project Sdk="Microsoft.NET.Sdk" DefaultTargets="Pack">
+
+  <PropertyGroup>
+    <TargetFramework>netstandard2.0</TargetFramework>
+    <PackageDescription>Microsoft.ML.Dnn contains APIs to do high level DNN training such as image classification.</PackageDescription>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="../Microsoft.ML/Microsoft.ML.nupkgproj" />
+  </ItemGroup>
+
+</Project>

pkg/Microsoft.ML.Dnn/Microsoft.ML.Dnn.symbols.nupkgproj

@@ -0,0 +1,5 @@
+<Project DefaultTargets="Pack">
+
+  <Import Project="Microsoft.ML.Dnn.nupkgproj" />
+
+</Project>

pkg/common/Resnet101V2.props

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup>
+    <Content Include="$(MSBuildThisFileDirectory)..\..\tools\DnnImageModels\Resnet101V2Tensorflow\*.*">
+      <Link>DnnImageModels\%(RecursiveDir)%(Filename)%(Extension)</Link>
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+  </ItemGroup>
+</Project>

src/Microsoft.ML.Core/Properties/AssemblyInfo.cs

@@ -37,6 +37,7 @@
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.StandardTrainers" + PublicKey.Value)]
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Sweeper" + PublicKey.Value)]
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.TensorFlow" + PublicKey.Value)]
+[assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Dnn" + PublicKey.Value)]
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.TimeSeries" + PublicKey.Value)]
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Transforms" + PublicKey.Value)]
 

src/Microsoft.ML.Data/Properties/AssemblyInfo.cs

@@ -36,6 +36,7 @@
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.StandardTrainers" + PublicKey.Value)]
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Sweeper" + PublicKey.Value)]
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.TensorFlow" + PublicKey.Value)]
+[assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Dnn" + PublicKey.Value)]
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.TimeSeries" + PublicKey.Value)]
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Transforms" + PublicKey.Value)]
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.DnnImageFeaturizer.AlexNet" + PublicKey.Value)]

src/Microsoft.ML.Dnn/AssemblyInfo.cs

@@ -0,0 +1,9 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Runtime.CompilerServices;
+using Microsoft.ML;
+
+[assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Tests" + PublicKey.TestValue)]
+[assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Tensorflow" + PublicKey.Value)]