diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ApplyONNXModelWithInMemoryImages.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ApplyONNXModelWithInMemoryImages.cs new file mode 100644 index 00000000000..430ad8fbfc8 --- /dev/null +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ApplyONNXModelWithInMemoryImages.cs @@ -0,0 +1,98 @@ +using System; +using System.Drawing; +using System.Linq; +using Microsoft.ML; +using Microsoft.ML.Data; +using Microsoft.ML.Transforms.Image; + +namespace Samples.Dynamic +{ + public static class ApplyOnnxModelWithInMemoryImages + { + // Example of applying ONNX transform on in-memory images. + public static void Example() + { + // Download the squeeznet image model from ONNX model zoo, version 1.2 + // https://github.com/onnx/models/tree/master/squeezenet or use + // Microsoft.ML.Onnx.TestModels nuget. + // It's a multiclass classifier. It consumes an input "data_0" and produces + // an output "softmaxout_1". + var modelPath = @"squeezenet\00000001\model.onnx"; + + // Create ML pipeline to score the data using OnnxScoringEstimator + var mlContext = new MLContext(); + + // Create in-memory data points. Its Image/Scores field is the input/output of the used ONNX model. + var dataPoints = new ImageDataPoint[] + { + new ImageDataPoint(Color.Red), + new ImageDataPoint(Color.Green) + }; + + // Convert training data to IDataView, the general data type used in ML.NET. + var dataView = mlContext.Data.LoadFromEnumerable(dataPoints); + + // Create a ML.NET pipeline which contains two steps. First, ExtractPixle is used to convert the 224x224 image to a 3x224x224 float tensor. + // Then the float tensor is fed into a ONNX model with an input called "data_0" and an output called "softmaxout_1". Note that "data_0" and + // "softmaxout_1" are model input and output names stored in the used ONNX model file. Users may need to inspect their own models to + // get the right input and output column names. + var pipeline = mlContext.Transforms.ExtractPixels("data_0", "Image") // Map column "Image" to column "data_0" + .Append(mlContext.Transforms.ApplyOnnxModel("softmaxout_1", "data_0", modelPath)); // Map column "data_0" to column "softmaxout_1" + var model = pipeline.Fit(dataView); + var onnx = model.Transform(dataView); + + // Convert IDataView back to IEnumerable so that user can inspect the output, column "softmaxout_1", of the ONNX transform. + // Note that Column "softmaxout_1" would be stored in ImageDataPont.Scores because the added attributed [ColumnName("softmaxout_1")] + // tells that ImageDataPont.Scores is equivalent to column "softmaxout_1". + var transformedDataPoints = mlContext.Data.CreateEnumerable(onnx, false).ToList(); + + // The scores are probabilities of all possible classes, so they should all be positive. + foreach (var dataPoint in transformedDataPoints) + { + var firstClassProb = dataPoint.Scores.First(); + var lastClassProb = dataPoint.Scores.Last(); + Console.WriteLine($"The probability of being the first class is {firstClassProb * 100}%."); + Console.WriteLine($"The probability of being the last class is {lastClassProb * 100}%."); + } + + // Expected output: + // The probability of being the first class is 0.002542659%. + // The probability of being the last class is 0.0292684%. + // The probability of being the first class is 0.02258059%. + // The probability of being the last class is 0.394428%. + } + + // This class is used in Example() to describe data points which will be consumed by ML.NET pipeline. + private class ImageDataPoint + { + // Height of Image. + private const int height = 224; + + // Width of Image. + private const int width = 224; + + // Image will be consumed by ONNX image multiclass classification model. + [ImageType(height, width)] + public Bitmap Image { get; set; } + + // Expected output of ONNX model. It contains probabilities of all classes. + // Note that the ColumnName below should match the output name in the used + // ONNX model file. + [ColumnName("softmaxout_1")] + public float[] Scores { get; set; } + + public ImageDataPoint() + { + Image = null; + } + + public ImageDataPoint(Color color) + { + Image = new Bitmap(width, height); + for (int i = 0; i < width; ++i) + for (int j = 0; j < height; ++j) + Image.SetPixel(i, j, color); + } + } + } +} diff --git a/src/Microsoft.ML.ImageAnalytics/ExtensionsCatalog.cs b/src/Microsoft.ML.ImageAnalytics/ExtensionsCatalog.cs index 78cbc748740..60e92e79a22 100644 --- a/src/Microsoft.ML.ImageAnalytics/ExtensionsCatalog.cs +++ b/src/Microsoft.ML.ImageAnalytics/ExtensionsCatalog.cs @@ -118,6 +118,7 @@ internal static ImageLoadingEstimator LoadImages(this TransformsCatalog catalog, /// /// /// public static ImagePixelExtractingEstimator ExtractPixels(this TransformsCatalog catalog, diff --git a/src/Microsoft.ML.OnnxTransformer/OnnxCatalog.cs b/src/Microsoft.ML.OnnxTransformer/OnnxCatalog.cs index 40717ec5903..c6136ed2e67 100644 --- a/src/Microsoft.ML.OnnxTransformer/OnnxCatalog.cs +++ b/src/Microsoft.ML.OnnxTransformer/OnnxCatalog.cs @@ -45,6 +45,13 @@ public static OnnxScoringEstimator ApplyOnnxModel(this TransformsCatalog catalog /// The path of the file containing the ONNX model. /// Optional GPU device ID to run execution on, to run on CPU. /// If GPU error, raise exception or fallback to CPU. + /// + /// + /// + /// + /// public static OnnxScoringEstimator ApplyOnnxModel(this TransformsCatalog catalog, string outputColumnName, string inputColumnName, diff --git a/test/Microsoft.ML.OnnxTransformerTest/OnnxTransformTests.cs b/test/Microsoft.ML.OnnxTransformerTest/OnnxTransformTests.cs index 7cae32ce029..722debce03b 100644 --- a/test/Microsoft.ML.OnnxTransformerTest/OnnxTransformTests.cs +++ b/test/Microsoft.ML.OnnxTransformerTest/OnnxTransformTests.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; +using System.Drawing; using System.IO; using System.Linq; using Microsoft.ML; @@ -14,6 +15,7 @@ using Microsoft.ML.StaticPipe; using Microsoft.ML.TestFramework.Attributes; using Microsoft.ML.Tools; +using Microsoft.ML.Transforms.Image; using Microsoft.ML.Transforms.StaticPipe; using Xunit; using Xunit.Abstractions; @@ -316,5 +318,85 @@ public void TestUnknownDimensions() Assert.Equal(0, predictions[1].argmax[0]); Assert.Equal(2, predictions[2].argmax[0]); } + + /// + /// This class is used in to describe data points which will be consumed by ML.NET pipeline. + /// + private class ImageDataPoint + { + /// + /// Height of . + /// + private const int height = 224; + + /// + /// Width of . + /// + private const int width = 224; + + /// + /// Image will be consumed by ONNX image multiclass classification model. + /// + [ImageType(height, width)] + public Bitmap Image { get; set; } + + /// + /// Output of ONNX model. It contains probabilities of all classes. + /// + [ColumnName("softmaxout_1")] + public float[] Scores { get; set; } + + public ImageDataPoint() + { + Image = null; + } + + public ImageDataPoint(Color color) + { + Image = new Bitmap(width, height); + for (int i = 0; i < width; ++i) + for (int j = 0; j < height; ++j) + Image.SetPixel(i, j, color); + } + } + + /// + /// Test applying ONNX transform on in-memory image. + /// + [OnnxFact] + public void OnnxModelInMemoryImage() + { + // Path of ONNX model. It's a multiclass classifier. It consumes an input "data_0" and produces an output "softmaxout_1". + var modelFile = "squeezenet/00000001/model.onnx"; + + // Create in-memory data points. Its Image/Scores field is the input/output of the used ONNX model. + var dataPoints = new ImageDataPoint[] + { + new ImageDataPoint(Color.Red), + new ImageDataPoint(Color.Green) + }; + + // Convert training data to IDataView, the general data type used in ML.NET. + var dataView = ML.Data.LoadFromEnumerable(dataPoints); + + // Create a ML.NET pipeline which contains two steps. First, ExtractPixle is used to convert the 224x224 image to a 3x224x224 float tensor. + // Then the float tensor is fed into a ONNX model with an input called "data_0" and an output called "softmaxout_1". Note that "data_0" and + // "softmaxout_1" are model input and output names stored in the used ONNX model file. Users may need to inspect their own models to + // get the right input and output column names. + var pipeline = ML.Transforms.ExtractPixels("data_0", "Image") // Map column "Image" to column "data_0" + .Append(ML.Transforms.ApplyOnnxModel("softmaxout_1", "data_0", modelFile)); // Map column "data_0" to column "softmaxout_1" + var model = pipeline.Fit(dataView); + var onnx = model.Transform(dataView); + + // Convert IDataView back to IEnumerable so that user can inspect the output, column "softmaxout_1", of the ONNX transform. + // Note that Column "softmaxout_1" would be stored in ImageDataPont.Scores because the added attributed [ColumnName("softmaxout_1")] + // tells that ImageDataPont.Scores is equivalent to column "softmaxout_1". + var transformedDataPoints = ML.Data.CreateEnumerable(onnx, false).ToList(); + + // The scores are probabilities of all possible classes, so they should all be positive. + foreach (var dataPoint in transformedDataPoints) + foreach (var score in dataPoint.Scores) + Assert.True(score > 0); + } } }