Skip to content

Samples and unit test for image-related transform estimators #3165

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Apr 3, 2019
Merged

Samples and unit test for image-related transform estimators #3165

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
44870c5
updating image samples
abgoswam Apr 2, 2019
d5b4451
fix review comments (print preview)
abgoswam Apr 2, 2019
50c0f8d
fix comments (minor nits)
abgoswam Apr 2, 2019
c8cbd15
fix review comments (whitespaces)
abgoswam Apr 2, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 22 additions & 11 deletions docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ImageAnalytics/ConvertToGrayScale.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using System.IO;
using System;
using System.IO;
using Microsoft.ML.Data;

namespace Microsoft.ML.Samples.Dynamic
Expand Down Expand Up @@ -40,20 +41,30 @@ public static void Example()
.Append(mlContext.Transforms.ConvertToGrayscale("Grayscale", "ImageObject"));

var transformedData = pipeline.Fit(data).Transform(data);

// The transformedData IDataView contains the loaded images column, and the grayscaled column.
// Preview of the transformedData

// Preview the transformedData.
var transformedDataPreview = transformedData.Preview();
PrintPreview(transformedDataPreview);
// ImagePath Name ImageObject Grayscale
// tomato.bmp tomato System.Drawing.Bitmap System.Drawing.Bitmap
// banana.jpg banana System.Drawing.Bitmap System.Drawing.Bitmap
// hotdog.jpg hotdog System.Drawing.Bitmap System.Drawing.Bitmap
// tomato.jpg tomato System.Drawing.Bitmap System.Drawing.Bitmap
}

// Preview of the content of the images.tsv file
// The actual images, in the Grayscale column are of type System.Drawing.Bitmap.
//
// ImagePath Name ImageObject Grayscale
// tomato.bmp tomato {System.Drawing.Bitmap} {System.Drawing.Bitmap}
// banana.jpg banana {System.Drawing.Bitmap} {System.Drawing.Bitmap}
// hotdog.jpg hotdog {System.Drawing.Bitmap} {System.Drawing.Bitmap}
// tomato.jpg tomato {System.Drawing.Bitmap} {System.Drawing.Bitmap}
private static void PrintPreview(DataDebuggerPreview data)
{
foreach (var colInfo in data.ColumnView)
Console.Write("{0,-25}", colInfo.Column.Name);

Console.WriteLine();
foreach (var row in data.RowView)
{
Copy link
Member

@sfilipi sfilipi Apr 2, 2019 •
edited by abgoswam
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

{ [](start = 12, length = 1)

omit :) #Resolved

foreach (var kvPair in row.Values)
Console.Write("{0,-25}", kvPair.Value);
Console.WriteLine();
}
}
}
}
80 changes: 80 additions & 0 deletions docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ImageAnalytics/ConvertToImage.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
using System;
using System.Collections.Generic;
using System.Linq;
using Microsoft.ML.Data;

namespace Microsoft.ML.Samples.Dynamic
{
public static class ConvertToImage
{
private const int imageHeight = 224;
private const int imageWidth = 224;
private const int numberOfChannels = 3;
private const int inputSize = imageHeight * imageWidth * numberOfChannels;

// Sample that shows how an input array (of doubles) can be used to interop with image related estimators in ML.NET.
public static void Example()
{
// Create a new ML context, for ML.NET operations. It can be used for exception tracking and logging,
// as well as the source of randomness.
var mlContext = new MLContext();

// Create a list of training data points.
var dataPoints = GenerateRandomDataPoints(4);

// Convert the list of data points to an IDataView object, which is consumable by ML.NET API.
var data = mlContext.Data.LoadFromEnumerable(dataPoints);

// Image loading pipeline.
var pipeline = mlContext.Transforms.ConvertToImage(imageHeight, imageWidth, "Image", "Features")
.Append(mlContext.Transforms.ExtractPixels("Pixels", "Image"));

var transformedData = pipeline.Fit(data).Transform(data);

// Preview the transformedData.
var transformedDataPreview = transformedData.Preview();
PrintPreview(transformedDataPreview);
// Features Image Pixels
// 185,209,196,142,52 System.Drawing.Bitmap 185,209,196,142,52
// 182,235,84,23,87 System.Drawing.Bitmap 182,235,84,23,87
// 192,214,247,22,38 System.Drawing.Bitmap 192,214,247,22,38
// 242,161,141,223,192 System.Drawing.Bitmap 242,161,141,223,192
}

private static void PrintPreview(DataDebuggerPreview data)
{
foreach (var colInfo in data.ColumnView)
Console.Write("{0,-25}", colInfo.Column.Name);

Console.WriteLine();
foreach (var row in data.RowView)
{
Copy link
Member

@sfilipi sfilipi Apr 2, 2019 •
edited by abgoswam
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

{ [](start = 11, length = 2)

omit #Resolved

foreach (var kvPair in row.Values)
{
if (kvPair.Key == "Pixels" || kvPair.Key == "Features")
{
var rawValues = ((VBuffer<float>)kvPair.Value).DenseValues().Take(5);
Console.Write("{0,-25}", string.Join(",", rawValues));
}
else
Console.Write("{0,-25}", kvPair.Value);
}
Console.WriteLine();
}
}

private class DataPoint
{
[VectorType(inputSize)]
Copy link
Member

@sfilipi sfilipi Apr 3, 2019 •
edited by abgoswam
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[VectorType(inputSize)] [](start = 11, length = 24)

is this annotation needed? #Resolved

Copy link
Member Author

@abgoswam abgoswam Apr 3, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yeah, this is indeed needed.

when fitting a model, we do some checks e.g. schema validation / size checks / whether the estimator works with fixed size or variable size etc

the VectorToImageConvertingEstimator works on fixed-size vector . without the annotation, the assumtion is that its a variable size vector , and the following check fails :

machinelearning/src/Microsoft.ML.ImageAnalytics/VectorToImageTransform.cs

Line 736 in a7bcdda

if (col.Kind != SchemaShape.Column.VectorKind.Vector || (col.ItemType != NumberDataViewType.Single && col.ItemType != NumberDataViewType.Double && col.ItemType != NumberDataViewType.Byte))

Unhandled Exception: System.ArgumentOutOfRangeException: Schema mismatch for input column 'Features': expected known-size vector of type Single, Double or Byte, got VarVector

In reply to: 271595813 [](ancestors = 271595813)

public float[] Features { get; set; }
}

Copy link
Member

@sfilipi sfilipi Apr 2, 2019 •
edited by abgoswam
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

no need for it. Users can add it #Resolved

private static IEnumerable<DataPoint> GenerateRandomDataPoints(int count, int seed = 0)
{
var random = new Random(seed);

for (int i = 0; i < count; i++)
yield return new DataPoint { Features = Enumerable.Repeat(0, inputSize).Select(x => (float)random.Next(0, 256)).ToArray() };
}
}
}
45 changes: 32 additions & 13 deletions docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ImageAnalytics/ExtractPixels.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
using System.IO;
using System;
using System.IO;
using System.Linq;
using Microsoft.ML.Data;

namespace Microsoft.ML.Samples.Dynamic
Expand Down Expand Up @@ -38,24 +40,41 @@ public static void Example()
var imagesFolder = Path.GetDirectoryName(imagesDataFile);
// Image loading pipeline.
var pipeline = mlContext.Transforms.LoadImages("ImageObject", imagesFolder, "ImagePath")
.Append(mlContext.Transforms.ResizeImages("ImageObject", imageWidth: 100, imageHeight: 100 ))
.Append(mlContext.Transforms.ExtractPixels("Pixels", "ImageObject"));

.Append(mlContext.Transforms.ResizeImages("ImageObjectResized", inputColumnName: "ImageObject", imageWidth: 100, imageHeight: 100))
.Append(mlContext.Transforms.ExtractPixels("Pixels", "ImageObjectResized"));

var transformedData = pipeline.Fit(data).Transform(data);

// The transformedData IDataView contains the loaded images now
//Preview of the transformedData
// Preview the transformedData.
var transformedDataPreview = transformedData.Preview();
PrintPreview(transformedDataPreview);
// ImagePath Name ImageObject ImageObjectResized Pixels
// tomato.bmp tomato System.Drawing.Bitmap System.Drawing.Bitmap 255,255,255,255,255...
// banana.jpg banana System.Drawing.Bitmap System.Drawing.Bitmap 255,255,255,255,255...
// hotdog.jpg hotdog System.Drawing.Bitmap System.Drawing.Bitmap 255,255,255,255,255...
// tomato.jpg tomato System.Drawing.Bitmap System.Drawing.Bitmap 255,255,255,255,255...
}

// Preview of the content of the images.tsv file
//
// ImagePath Name ImageObject "Pixels"
Copy link
Member

@sfilipi sfilipi Apr 2, 2019 •
edited by abgoswam
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

// ImagePath Name ImageObject [](start = 11, length = 41)

maybe keep the headers/column names, even if they don't print out. Or print them separately. #Resolved

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fixed

In reply to: 271136252 [](ancestors = 271136252)

// tomato.bmp tomato {System.Drawing.Bitmap} [ 255, 255, 255, ..... 232, 243, 226, ...
// banana.jpg banana {System.Drawing.Bitmap} [ 255, 255, 255, ..... 90, 54, 43, ...
// hotdog.jpg hotdog {System.Drawing.Bitmap} [ 255, 255, 255, ..... 132, 143, 126, ...
// tomato.jpg tomato {System.Drawing.Bitmap} [ 255, 255, 255, ..... 16, 21, 23, ...
private static void PrintPreview(DataDebuggerPreview data)
{
foreach (var colInfo in data.ColumnView)
Console.Write("{0,-25}", colInfo.Column.Name);

Console.WriteLine();
foreach (var row in data.RowView)
{
foreach (var kvPair in row.Values)
{
if (kvPair.Key == "Pixels")
{
var pixels = ((VBuffer<float>)kvPair.Value).DenseValues().Take(5);
Console.Write("{0}...", string.Join(",", pixels));
}
else
Console.Write("{0,-25}", kvPair.Value);
}
Console.WriteLine();
}
}
}
}
36 changes: 24 additions & 12 deletions docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ImageAnalytics/LoadImages.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using System.IO;
using System;
using System.IO;
using Microsoft.ML.Data;

namespace Microsoft.ML.Samples.Dynamic
Expand Down Expand Up @@ -36,22 +37,33 @@ public static void Example()

var imagesFolder = Path.GetDirectoryName(imagesDataFile);
// Image loading pipeline.
var pipeline = mlContext.Transforms.LoadImages("ImageReal", imagesFolder, "ImagePath");
var pipeline = mlContext.Transforms.LoadImages("ImageObject", imagesFolder, "ImagePath");

var transformedData = pipeline.Fit(data).Transform(data);
// The transformedData IDataView contains the loaded images now.

// The transformedData IDataView contains the loaded images now
//Preview of the transformedData
// Preview the transformedData.
var transformedDataPreview = transformedData.Preview();
PrintPreview(transformedDataPreview);
// ImagePath Name ImageObject
// tomato.bmp tomato System.Drawing.Bitmap
// banana.jpg banana System.Drawing.Bitmap
// hotdog.jpg hotdog System.Drawing.Bitmap
// tomato.jpg tomato System.Drawing.Bitmap
}

// Preview of the content of the images.tsv file
// The actual images, in the ImageReal column are of type System.Drawing.Bitmap.
//
// ImagePath Name ImageReal
// tomato.bmp tomato {System.Drawing.Bitmap}
// banana.jpg banana {System.Drawing.Bitmap}
// hotdog.jpg hotdog {System.Drawing.Bitmap}
// tomato.jpg tomato {System.Drawing.Bitmap}
private static void PrintPreview(DataDebuggerPreview data)
{
foreach (var colInfo in data.ColumnView)
Console.Write("{0,-25}", colInfo.Column.Name);

Console.WriteLine();
foreach (var row in data.RowView)
{
foreach (var kvPair in row.Values)
Console.Write("{0,-25}", kvPair.Value);
Console.WriteLine();
}
}
}
}
38 changes: 24 additions & 14 deletions docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ImageAnalytics/ResizeImages.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using System.IO;
using System;
using System.IO;
using Microsoft.ML.Data;

namespace Microsoft.ML.Samples.Dynamic
Expand Down Expand Up @@ -36,25 +37,34 @@ public static void Example()

var imagesFolder = Path.GetDirectoryName(imagesDataFile);
// Image loading pipeline.
var pipeline = mlContext.Transforms.LoadImages("ImageReal", imagesFolder, "ImagePath")
.Append(mlContext.Transforms.ResizeImages("ImageReal", imageWidth: 100, imageHeight: 100));

var pipeline = mlContext.Transforms.LoadImages("ImageObject", imagesFolder, "ImagePath")
.Append(mlContext.Transforms.ResizeImages("ImageObjectResized", inputColumnName: "ImageObject", imageWidth: 100, imageHeight: 100));

var transformedData = pipeline.Fit(data).Transform(data);
// The transformedData IDataView contains the resized images now.

// The transformedData IDataView contains the loaded images now
//Preview of the transformedData
// Preview the transformedData.
var transformedDataPreview = transformedData.Preview();
PrintPreview(transformedDataPreview);
// ImagePath Name ImageObject ImageObjectResized
// tomato.bmp tomato System.Drawing.Bitmap System.Drawing.Bitmap
// banana.jpg banana System.Drawing.Bitmap System.Drawing.Bitmap
// hotdog.jpg hotdog System.Drawing.Bitmap System.Drawing.Bitmap
// tomato.jpg tomato System.Drawing.Bitmap System.Drawing.Bitmap
}

// Preview of the content of the images.tsv file
// The actual images, in the ImageReal column are of type System.Drawing.Bitmap.
//
// ImagePath Name ImageReal
// tomato.bmp tomato {System.Drawing.Bitmap}
// banana.jpg banana {System.Drawing.Bitmap}
// hotdog.jpg hotdog {System.Drawing.Bitmap}
// tomato.jpg tomato {System.Drawing.Bitmap}
private static void PrintPreview(DataDebuggerPreview data)
{
foreach (var colInfo in data.ColumnView)
Console.Write("{0,-25}", colInfo.Column.Name);

Console.WriteLine();
foreach (var row in data.RowView)
{
foreach (var kvPair in row.Values)
Console.Write("{0,-25}", kvPair.Value);
Console.WriteLine();
}
}
}
}
6 changes: 6 additions & 0 deletions src/Microsoft.ML.ImageAnalytics/ExtensionsCatalog.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,12 @@ internal static VectorToImageConvertingEstimator ConvertToImage(this TransformsC
/// <param name="defaultRed">Default value for red color, would be overriden if <paramref name="colorsPresent"/> contains <see cref="ImagePixelExtractingEstimator.ColorBits.Red"/>.</param>
/// <param name="defaultGreen">Default value for grenn color, would be overriden if <paramref name="colorsPresent"/> contains <see cref="ImagePixelExtractingEstimator.ColorBits.Green"/>.</param>
/// <param name="defaultBlue">Default value for blue color, would be overriden if <paramref name="colorsPresent"/> contains <see cref="ImagePixelExtractingEstimator.ColorBits.Blue"/>.</param>
/// <example>
/// <format type="text/markdown">
/// <![CDATA[
/// [!code-csharp[ConvertToImage](~/../docs/samples/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/ImageAnalytics/ConvertToImage.cs)]
/// ]]></format>
/// </example>
public static VectorToImageConvertingEstimator ConvertToImage(this TransformsCatalog catalog, int imageHeight, int imageWidth, string outputColumnName, string inputColumnName = null,
ImagePixelExtractingEstimator.ColorBits colorsPresent = ImagePixelExtractingEstimator.Defaults.Colors,
ImagePixelExtractingEstimator.ColorsOrder orderOfColors = ImagePixelExtractingEstimator.Defaults.Order,
Expand Down
39 changes: 39 additions & 0 deletions test/Microsoft.ML.Tests/ImagesTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
// See the LICENSE file in the project root for more information.

using System;
using System.Collections.Generic;
using System.Drawing;
using System.IO;
using System.Linq;
Expand Down Expand Up @@ -754,5 +755,43 @@ public void ImageResizerTransformResizingModeFill()

Done();
}

[Fact]
public void TestConvertToImage()
{
var mlContext = new MLContext(0);

// Create a list of training data points.
var dataPoints = GenerateRandomDataPoints(10);

// Convert the list of data points to an IDataView object, which is consumable by ML.NET API.
var data = mlContext.Data.LoadFromEnumerable(dataPoints);

var pipeline = mlContext.Transforms.ConvertToImage(224, 224, "Features");

TestEstimatorCore(pipeline, data);
Done();
}

private const int inputSize = 3 * 224 * 224;

private static IEnumerable<DataPoint> GenerateRandomDataPoints(int count, int seed = 0)
{
var random = new Random(seed);

for (int i = 0; i < count; i++)
{
yield return new DataPoint
{
Features = Enumerable.Repeat(0, inputSize).Select(x => random.NextDouble()*100).ToArray()
};
}
}

private class DataPoint
{
[VectorType(inputSize)]
public double[] Features { get; set; }
}
}
}