add image featurizer to AutoFeaturizer

src/Microsoft.ML.AutoML/API/AutoCatalog.cs

@@ -587,6 +587,44 @@ internal SweepableEstimator[] CatalogFeaturizer(string[] outputColumnNames, stri
             return new SweepableEstimator[] { SweepableEstimatorFactory.CreateOneHotEncoding(option), SweepableEstimatorFactory.CreateOneHotHashEncoding(option) };
         }
 
+        internal MultiModelPipeline ImagePathFeaturizer(string outputColumnName, string inputColumnName)
+        {
+            // load image => resize image (224, 224) => extract pixels => dnn featurizer
+            var loadImageOption = new LoadImageOption
+            {
+                ImageFolder = null,
+                InputColumnName = inputColumnName,
+                OutputColumnName = outputColumnName,
+            };
+
+            var resizeImageOption = new ResizeImageOption
+            {
+                ImageHeight = 224,
+                ImageWidth = 224,
+                InputColumnName = inputColumnName,
+                OutputColumnName = outputColumnName,
+            };
+
+            var extractPixelOption = new ExtractPixelsOption
+            {
+                InputColumnName = inputColumnName,
+                OutputColumnName = outputColumnName,
+            };
+
+            var dnnFeaturizerOption = new DnnFeaturizerImageOption
+            {
+                InputColumnName = inputColumnName,
+                OutputColumnName = outputColumnName,
+            };
+
+            var pipeline = new MultiModelPipeline();
+
+            return pipeline.Append(SweepableEstimatorFactory.CreateLoadImages(loadImageOption))
+                        .Append(SweepableEstimatorFactory.CreateResizeImages(resizeImageOption))
+                        .Append(SweepableEstimatorFactory.CreateExtractPixels(extractPixelOption))
+                        .Append(SweepableEstimatorFactory.CreateDnnFeaturizerImage(dnnFeaturizerOption));
+        }
+
         /// <summary>
         /// Create a single featurize pipeline according to <paramref name="data"/>. This function will collect all columns in <paramref name="data"/> and not in <paramref name="excludeColumns"/>,
         /// featurizing them using <see cref="CatalogFeaturizer(string[], string[])"/>, <see cref="NumericFeaturizer(string[], string[])"/> or <see cref="TextFeaturizer(string, string)"/>. And combine
@@ -596,9 +634,10 @@ internal SweepableEstimator[] CatalogFeaturizer(string[] outputColumnNames, stri
         /// <param name="catalogColumns">columns that should be treated as catalog. If not specified, it will automatically infer if a column is catalog or not.</param>
         /// <param name="numericColumns">columns that should be treated as numeric. If not specified, it will automatically infer if a column is catalog or not.</param>
         /// <param name="textColumns">columns that should be treated as text. If not specified, it will automatically infer if a column is catalog or not.</param>
+        /// <param name="imagePathColumns">columns that should be treated as image path. If not specified, it will automatically infer if a column is catalog or not.</param>
         /// <param name="outputColumnName">output feature column.</param>
         /// <param name="excludeColumns">columns that won't be included when featurizing, like label</param>
-        public MultiModelPipeline Featurizer(IDataView data, string outputColumnName = "Features", string[] catalogColumns = null, string[] numericColumns = null, string[] textColumns = null, string[] excludeColumns = null)
+        public MultiModelPipeline Featurizer(IDataView data, string outputColumnName = "Features", string[] catalogColumns = null, string[] numericColumns = null, string[] textColumns = null, string[] imagePathColumns = null, string[] excludeColumns = null)
         {
             Contracts.CheckValue(data, nameof(data));
 
@@ -646,6 +685,14 @@ public MultiModelPipeline Featurizer(IDataView data, string outputColumnName = "
                 }
             }
 
+            if (imagePathColumns != null)
+            {
+                foreach (var column in imagePathColumns)
+                {
+                    columnInfo.ImagePathColumnNames.Add(column);
+                }
+            }
+
             return this.Featurizer(data, columnInfo, outputColumnName);
         }
 
@@ -667,9 +714,11 @@ public MultiModelPipeline Featurizer(IDataView data, ColumnInformation columnInf
             var textFeatures = columnPurposes.Where(c => c.Purpose == ColumnPurpose.TextFeature);
             var numericFeatures = columnPurposes.Where(c => c.Purpose == ColumnPurpose.NumericFeature);
             var catalogFeatures = columnPurposes.Where(c => c.Purpose == ColumnPurpose.CategoricalFeature);
+            var imagePathFeatures = columnPurposes.Where(c => c.Purpose == ColumnPurpose.ImagePath);
             var textFeatureColumnNames = textFeatures.Select(c => data.Schema[c.ColumnIndex].Name).ToArray();
             var numericFeatureColumnNames = numericFeatures.Select(c => data.Schema[c.ColumnIndex].Name).ToArray();
             var catalogFeatureColumnNames = catalogFeatures.Select(c => data.Schema[c.ColumnIndex].Name).ToArray();
+            var imagePathColumnNames = imagePathFeatures.Select(c => data.Schema[c.ColumnIndex].Name).ToArray();
 
             var pipeline = new MultiModelPipeline();
             if (numericFeatureColumnNames.Length > 0)
@@ -682,14 +731,19 @@ public MultiModelPipeline Featurizer(IDataView data, ColumnInformation columnInf
                 pipeline = pipeline.Append(this.CatalogFeaturizer(catalogFeatureColumnNames, catalogFeatureColumnNames));
             }
 
+            foreach (var imagePathColumn in imagePathColumnNames)
+            {
+                pipeline = pipeline.Append(this.ImagePathFeaturizer(imagePathColumn, imagePathColumn));
+            }
+
             foreach (var textColumn in textFeatureColumnNames)
             {
                 pipeline = pipeline.Append(this.TextFeaturizer(textColumn, textColumn));
             }
 
             var option = new ConcatOption
             {
-                InputColumnNames = textFeatureColumnNames.Concat(numericFeatureColumnNames).Concat(catalogFeatureColumnNames).ToArray(),
+                InputColumnNames = textFeatureColumnNames.Concat(numericFeatureColumnNames).Concat(catalogFeatureColumnNames).Concat(imagePathColumnNames).ToArray(),
                 OutputColumnName = outputColumnName,
             };
 

src/Microsoft.ML.AutoML/CodeGen/dnn_featurizer_image_search_space.json

@@ -0,0 +1,22 @@
+{
+  "$schema": "./search-space-schema.json#",
+  "name": "dnn_featurizer_image_option",
+  "search_space": [
+    {
+      "name": "OutputColumnName",
+      "type": "string"
+    },
+    {
+      "name": "InputColumnName",
+      "type": "string"
+    },
+    {
+      "name": "ModelFactory",
+      "type": "dnnModelFactory",
+      "default": "resnet_18",
+      "search_space": [
+        "alexnet", "resnet_101", "resnet_18", "resnet_50"
+      ]
+    }
+  ]
+}

src/Microsoft.ML.AutoML/CodeGen/estimator-schema.json

@@ -68,6 +68,7 @@
               "ApplyOnnxModel",
               "ResizeImages",
               "ExtractPixels",
+              "DnnFeaturizerImage",
               "Naive",
               "ForecastBySsa"
             ]
@@ -180,22 +181,12 @@
                     "confidenceLowerBoundColumn",
                     "confidenceUpperBoundColumn",
                     "confidenceLevel",
-                    "variableHorizon"
+                    "variableHorizon",
+                    "modelFactory"
                   ]
                 },
                 "argumentType": {
-                  "type": "string",
-                  "enum": [
-                    "integer",
-                    "float",
-                    "double",
-                    "string",
-                    "boolean",
-                    "resizingKind",
-                    "colorBits",
-                    "colorsOrder",
-                    "anchor"
-                  ]
+                  "$ref": "search-space-schema.json#/definitions/option_type"
                 }
               }
             }

src/Microsoft.ML.AutoML/CodeGen/search-space-schema.json

@@ -2,6 +2,29 @@
   "$schema": "http://json-schema.org/draft-04/schema",
   "title": "Search Space",
   "definitions": {
+    "boolArray": {
+      "type": "array",
+      "items": { "type": "boolean" }
+    },
+    "intArray": {
+      "type": "array",
+      "items": { "type": "integer" }
+    },
+    "dnnModelFactoryArray": {
+      "type": "array",
+      "items": {
+        "$ref": "#/definitions/dnnModelFactoryType"
+      }
+    },
+    "dnnModelFactoryType": {
+      "type": "string",
+      "enum": [
+        "resnet_18",
+        "resnet_50",
+        "resnet_101",
+        "alexnet"
+      ]
+    },
     "range": {
       "type": "object",
       "properties": {
@@ -12,18 +35,17 @@
       "required": [ "min", "max" ]
     },
     "choice": {
-      "type": "object",
-      "properties": {
-        "value": {
-          "oneOf": [
-            { "type": "string" },
-            { "type": "number" },
-            { "type": "integer" },
-            { "type": "boolean" }
-          ]
+      "oneOf": [
+        {
+          "$ref": "#/definitions/intArray"
+        },
+        {
+          "$ref": "#/definitions/dnnModelFactoryArray"
+        },
+        {
+          "$ref": "#/definitions/boolArray"
         }
-      },
-      "required": [ "value" ]
+      ]
     },
     "option": {
       "type": "object",
@@ -86,7 +108,8 @@
         "extract_pixels_option",
         "load_image_option",
         "image_classification_option",
-        "matrix_factorization_option"
+        "matrix_factorization_option",
+        "dnn_featurizer_image_option"
       ]
     },
     "option_name": {
@@ -130,7 +153,8 @@
         "ApproximationRank",
         "NumberOfIterations",
         "Quiet",
-        "OutputAsFloatArray"
+        "OutputAsFloatArray",
+        "ModelFactory"
       ]
     },
     "option_type": {
@@ -145,7 +169,8 @@
         "resizingKind",
         "colorBits",
         "colorsOrder",
-        "anchor"
+        "anchor",
+        "dnnModelFactory"
       ]
     }
   },

src/Microsoft.ML.AutoML/CodeGen/transformer-estimators.json

@@ -321,6 +321,27 @@
       "nugetDependencies": [ "Microsoft.ML", "Microsoft.ML.ImageAnalytics" ],
       "usingStatements": [ "Microsoft.ML" ],
       "searchOption": "load_image_option"
+    },
+    {
+      "functionName": "DnnFeaturizerImage",
+      "estimatorTypes": [ "Transforms" ],
+      "arguments": [
+        {
+          "argumentName": "outputColumnName",
+          "argumentType": "string"
+        },
+        {
+          "argumentName": "inputColumnName",
+          "argumentType": "string"
+        },
+        {
+          "argumentName": "modelFactory",
+          "argumentType": "dnnModelFactory"
+        }
+      ],
+      "nugetDependencies": [ "Microsoft.ML.OnnxTransformer", "Microsoft.ML.OnnxRuntime" ],
+      "usingStatements": [ "Microsoft.ML" ],
+      "searchOption": "dnn_featurizer_image_option"
     }
   ]
 }

src/Microsoft.ML.AutoML/Microsoft.ML.AutoML.csproj

@@ -35,6 +35,10 @@
       <PrivateAssets>all</PrivateAssets>
     </ProjectReference>
     <ProjectReference Include="..\Microsoft.ML.CpuMath\Microsoft.ML.CpuMath.csproj" />
+    <ProjectReference Include="..\Microsoft.ML.DnnImageFeaturizer.AlexNet\Microsoft.ML.DnnImageFeaturizer.AlexNet.csproj" />
+    <ProjectReference Include="..\Microsoft.ML.DnnImageFeaturizer.ResNet101\Microsoft.ML.DnnImageFeaturizer.ResNet101.csproj" />
+    <ProjectReference Include="..\Microsoft.ML.DnnImageFeaturizer.ResNet18\Microsoft.ML.DnnImageFeaturizer.ResNet18.csproj" />
+    <ProjectReference Include="..\Microsoft.ML.DnnImageFeaturizer.ResNet50\Microsoft.ML.DnnImageFeaturizer.ResNet50.csproj" />
     <ProjectReference Include="..\Microsoft.ML.OnnxTransformer\Microsoft.ML.OnnxTransformer.csproj" />
     <ProjectReference Include="..\Microsoft.ML.SearchSpace\Microsoft.ML.SearchSpace.csproj">
       <PrivateAssets>all</PrivateAssets>

src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/Images.cs

@@ -1,7 +1,7 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
-
+using System;
 namespace Microsoft.ML.AutoML.CodeGen
 {
     internal partial class LoadImages
@@ -44,4 +44,28 @@ public override IEstimator<ITransformer> BuildFromOption(MLContext context, Imag
             return context.MulticlassClassification.Trainers.ImageClassification(param.LabelColumnName, param.FeatureColumnName, param.ScoreColumnName);
         }
     }
+
+    internal partial class DnnFeaturizerImage
+    {
+        public override IEstimator<ITransformer> BuildFromOption(MLContext context, DnnFeaturizerImageOption param)
+        {
+            switch (param.ModelFactory)
+            {
+                case "resnet_50":
+                    return context.Transforms.DnnFeaturizeImage(param.OutputColumnName,
+                        m => m.ModelSelector.ResNet50(context, param.OutputColumnName, param.InputColumnName), param.InputColumnName);
+                case "resnet_18":
+                    return context.Transforms.DnnFeaturizeImage(param.OutputColumnName,
+                        m => m.ModelSelector.ResNet18(context, param.OutputColumnName, param.InputColumnName), param.InputColumnName);
+                case "resnet_101":
+                    return context.Transforms.DnnFeaturizeImage(param.OutputColumnName,
+                        m => m.ModelSelector.ResNet101(context, param.OutputColumnName, param.InputColumnName), param.InputColumnName);
+                case "alexnet":
+                    return context.Transforms.DnnFeaturizeImage(param.OutputColumnName,
+                        m => m.ModelSelector.AlexNet(context, param.OutputColumnName, param.InputColumnName), param.InputColumnName);
+                default:
+                    throw new NotImplementedException();
+            }
+        }
+    }
 }

test/Microsoft.ML.AutoML.Tests/ApprovalTests/AutoFeaturizerTests.AutoFeaturizer_image_test.approved.txt

@@ -0,0 +1,50 @@
+{
+  "schema": "e0 * e1 * e2 * e3 * e4",
+  "estimators": {
+    "e0": {
+      "estimatorType": "LoadImages",
+      "parameter": {
+        "OutputColumnName": "ImagePath",
+        "InputColumnName": "ImagePath"
+      }
+    },
+    "e1": {
+      "estimatorType": "ResizeImages",
+      "parameter": {
+        "OutputColumnName": "ImagePath",
+        "InputColumnName": "ImagePath",
+        "ImageHeight": 224,
+        "ImageWidth": 224,
+        "CropAnchor": "Center",
+        "Resizing": "Fill"
+      }
+    },
+    "e2": {
+      "estimatorType": "ExtractPixels",
+      "parameter": {
+        "OutputColumnName": "ImagePath",
+        "InputColumnName": "ImagePath",
+        "ColorsToExtract": "Rgb",
+        "OrderOfExtraction": "ARGB",
+        "OutputAsFloatArray": true
+      }
+    },
+    "e3": {
+      "estimatorType": "DnnFeaturizerImage",
+      "parameter": {
+        "OutputColumnName": "ImagePath",
+        "InputColumnName": "ImagePath",
+        "ModelFactory": "resnet_18"
+      }
+    },
+    "e4": {
+      "estimatorType": "Concatenate",
+      "parameter": {
+        "InputColumnNames": [
+          "ImagePath"
+        ],
+        "OutputColumnName": "Features"
+      }
+    }
+  }
+}