AutoML Add Recommendation Task

docs/samples/Microsoft.ML.AutoML.Samples/DataStructures/Movie.cs

@@ -0,0 +1,22 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Microsoft.ML.Data;
+
+namespace Microsoft.ML.AutoML.Samples.DataStructures
+{
+    public class Movie
+    {
+        [LoadColumn(0)]
+        public string userId;
+
+        [LoadColumn(1)]
+        public string movieId;
+
+        [LoadColumn(2)]
+        public float rating;
+
+        [LoadColumn(3)]
+        public float timestamp;
+    }
+}

docs/samples/Microsoft.ML.AutoML.Samples/Microsoft.ML.AutoML.Samples.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
@@ -7,6 +7,7 @@
 
   <ItemGroup>
     <ProjectReference Include="..\..\..\src\Microsoft.ML.AutoML\Microsoft.ML.AutoML.csproj" />
+    <NativeAssemblyReference Include="MatrixFactorizationNative" />
   </ItemGroup>
 
 </Project>

docs/samples/Microsoft.ML.AutoML.Samples/Program.cs

@@ -8,6 +8,9 @@ public static void Main(string[] args)
         {
             try
             {
+                RecommendationExperiment.Run();
+                Console.Clear();
+
                 RegressionExperiment.Run();
                 Console.Clear();
 

docs/samples/Microsoft.ML.AutoML.Samples/RecommendationExperiment.cs

@@ -0,0 +1,79 @@
+using System;
+using System.IO;
+using System.Linq;
+using Microsoft.ML.AutoML;
+using Microsoft.ML.AutoML.Samples.DataStructures;
+using Microsoft.ML.Data;
+
+namespace Microsoft.ML.AutoML.Samples
+{
+    public static class RecommendationExperiment
+    {
+        private static string TrainDataPath = @"C:\Users\xiaoyuz\Desktop\machinelearning-samples\datasets\recommendation-ratings-train.csv";
+        private static string TestDataPath = @"C:\Users\xiaoyuz\Desktop\machinelearning-samples\datasets\recommendation-ratings-test.csv";
+        private static string ModelPath = @"C:\Users\xiaoyuz\source\test\recommendation.zip";
+        private static string LabelColumnName = "rating";
+        private static uint ExperimentTime = 60;
+
+        public static void Run()
+        {
+            MLContext mlContext = new MLContext();
+
+            // STEP 1: Load data
+            IDataView trainDataView = mlContext.Data.LoadFromTextFile<Movie>(TrainDataPath, hasHeader: true, separatorChar: ',');
+            IDataView testDataView = mlContext.Data.LoadFromTextFile<Movie>(TestDataPath, hasHeader: true, separatorChar: ',');
+
+            var settings = new RecommendationExperimentSettings(RecommendationExperimentScenario.MF, "userId", "movieId");
+            var inputColumnInformation = new ColumnInformation();
+            inputColumnInformation.LabelCategoricalColumnNames.Add("movieId");
+            inputColumnInformation.LabelCategoricalColumnNames.Add("userId");
+            inputColumnInformation.LabelColumnName = "rating";
+
+            // STEP 2: Run AutoML experiment
+            Console.WriteLine($"Running AutoML regression experiment for {ExperimentTime} seconds...");
+            ExperimentResult<RegressionMetrics> experimentResult = mlContext.Auto()
+                .CreateRecommendationExperiment(settings)
+                .Execute(trainDataView, inputColumnInformation);
+
+            // STEP 3: Print metric from best model
+            RunDetail<RegressionMetrics> bestRun = experimentResult.BestRun;
+            Console.WriteLine($"Total models produced: {experimentResult.RunDetails.Count()}");
+            Console.WriteLine($"Best model's trainer: {bestRun.TrainerName}");
+            Console.WriteLine($"Metrics of best model from validation data --");
+            PrintMetrics(bestRun.ValidationMetrics);
+
+            // STEP 5: Evaluate test data
+            IDataView testDataViewWithBestScore = bestRun.Model.Transform(testDataView);
+            RegressionMetrics testMetrics = mlContext.Regression.Evaluate(testDataViewWithBestScore, labelColumnName: LabelColumnName);
+            Console.WriteLine($"Metrics of best model on test data --");
+            PrintMetrics(testMetrics);
+
+            // STEP 6: Save the best model for later deployment and inferencing
+            using (FileStream fs = File.Create(ModelPath))
+                mlContext.Model.Save(bestRun.Model, trainDataView.Schema, fs);
+
+            // STEP 7: Create prediction engine from the best trained model
+            var predictionEngine = mlContext.Model.CreatePredictionEngine<Movie, TaxiTripFarePrediction>(bestRun.Model);
+
+            // STEP 8: Initialize a new test taxi trip, and get the predicted fare
+            var testTaxiTrip = new Movie
+            {
+                userId="1",
+                movieId = "1097",
+            };
+            var prediction = predictionEngine.Predict(testTaxiTrip);
+            Console.WriteLine($"Predicted fare for test taxi trip: {prediction.FareAmount}");
+
+            Console.WriteLine("Press any key to continue...");
+            Console.ReadKey();
+        }
+
+        private static void PrintMetrics(RegressionMetrics metrics)
+        {
+            Console.WriteLine($"MeanAbsoluteError: {metrics.MeanAbsoluteError}");
+            Console.WriteLine($"MeanSquaredError: {metrics.MeanSquaredError}");
+            Console.WriteLine($"RootMeanSquaredError: {metrics.RootMeanSquaredError}");
+            Console.WriteLine($"RSquared: {metrics.RSquared}");
+        }
+    }
+}

src/Microsoft.ML.AutoML/API/AutoCatalog.cs

@@ -2,6 +2,7 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
+using System.Collections.Generic;
 using Microsoft.ML.Data;
 
 namespace Microsoft.ML.AutoML
@@ -13,6 +14,11 @@ public sealed class AutoCatalog
     {
         private readonly MLContext _context;
 
+        /// <summary>
+        /// save some intermediate value
+        /// </summary>
+        public static Dictionary<string, object> ValuePairs { get; set; } = new Dictionary<string, object>();
+
         internal AutoCatalog(MLContext context)
         {
             _context = context;
@@ -123,6 +129,11 @@ public MulticlassClassificationExperiment CreateMulticlassClassificationExperime
             return new MulticlassClassificationExperiment(_context, experimentSettings);
         }
 
+        public RecommendationExperiment CreateRecommendationExperiment(RecommendationExperimentSettings experimentSettings)
+        {
+            return new RecommendationExperiment(_context, experimentSettings);
+        }
+
         /// <summary>
         /// Infers information about the columns of a dataset in a file located at <paramref name="path"/>.
         /// </summary>

src/Microsoft.ML.AutoML/API/ColumnInference.cs

@@ -2,6 +2,7 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
+using System.Collections;
 using System.Collections.Generic;
 using System.Collections.ObjectModel;
 using Microsoft.ML.Data;
@@ -76,6 +77,15 @@ public sealed class ColumnInformation
         /// </remarks>
         public ICollection<string> CategoricalColumnNames { get; }
 
+        /// <summary>
+        /// The dataset columns that are LabelFeature.
+        /// </summary>
+        /// <remarks>
+        /// It's quite similar with categorical feature, but it require ValueToKey converter instead of OneHotEncoding.
+        /// This column purpose can only be pre-set in this place, instead of being infered.
+        /// </remarks>
+        public ICollection<string> LabelCategoricalColumnNames { get; }
+
         /// <summary>
         /// The dataset columns that are numeric.
         /// </summary>
@@ -98,6 +108,7 @@ public ColumnInformation()
         {
             LabelColumnName = DefaultColumnNames.Label;
             CategoricalColumnNames = new Collection<string>();
+            LabelCategoricalColumnNames = new Collection<string>();
             NumericColumnNames = new Collection<string>();
             TextColumnNames = new Collection<string>();
             IgnoredColumnNames = new Collection<string>();

src/Microsoft.ML.AutoML/API/RecommendationExperiment.cs

@@ -0,0 +1,74 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using Microsoft.ML.Data;
+using Microsoft.ML.Trainers;
+
+namespace Microsoft.ML.AutoML
+{
+    public enum RecommendationExperimentScenario
+    {
+        MF,
+    }
+
+    public sealed class RecommendationExperimentSettings : ExperimentSettings
+    {
+        public RecommendationExperimentScenario Scenerio { get; set; }
+
+        public string MatrixColumnIndexColumnName { get; set; }
+
+        public string MatrixRowIndexColumnName { get; set; }
+
+        // We can use RegressionMetric as evaluation Metric
+        public RegressionMetric OptimizingMetric { get; set; }
+
+        public ICollection<RecommendationTrainer> Trainers { get; }
+
+        public RecommendationExperimentSettings(RecommendationExperimentScenario scenario, string columnIndexName, string rowIndexName)
+            : this()
+        {
+            if(scenario == RecommendationExperimentScenario.MF)
+            {
+                AutoCatalog.ValuePairs[nameof(MatrixFactorizationTrainer.Options.MatrixColumnIndexColumnName)] = columnIndexName;
+                AutoCatalog.ValuePairs[nameof(MatrixFactorizationTrainer.Options.MatrixRowIndexColumnName)] = rowIndexName;
+                return;
+            }
+            throw new NotImplementedException();
+        }
+
+        private RecommendationExperimentSettings()
+        {
+            OptimizingMetric = RegressionMetric.RSquared;
+            Trainers = Enum.GetValues(typeof(RecommendationTrainer)).OfType<RecommendationTrainer>().ToList();
+        }
+    }
+
+    public enum RecommendationTrainer
+    {
+        MatrixFactorization,
+    }
+
+    public sealed class RecommendationExperiment : ExperimentBase<RegressionMetrics, RecommendationExperimentSettings>
+    {
+        internal RecommendationExperiment(MLContext context, RecommendationExperimentSettings settings)
+            : base(context,
+                  new RegressionMetricsAgent(context, settings.OptimizingMetric),
+                  new OptimizingMetricInfo(settings.OptimizingMetric),
+                  settings,
+                  TaskKind.Recommendation,
+                  TrainerExtensionUtil.GetTrainerNames(settings.Trainers))
+        {
+        }
+        private protected override CrossValidationRunDetail<RegressionMetrics> GetBestCrossValRun(IEnumerable<CrossValidationRunDetail<RegressionMetrics>> results)
+        {
+            return BestResultUtil.GetBestRun(results, MetricsAgent, OptimizingMetricInfo.IsMaximizing);
+        }
+
+        private protected override RunDetail<RegressionMetrics> GetBestRun(IEnumerable<RunDetail<RegressionMetrics>> results)
+        {
+            return BestResultUtil.GetBestRun(results, MetricsAgent, OptimizingMetricInfo.IsMaximizing);
+        }
+    }
+}

src/Microsoft.ML.AutoML/Assembly.cs

@@ -11,3 +11,5 @@
 [assembly: InternalsVisibleTo("Benchmark, PublicKey=00240000048000009400000006020000002400005253413100040000010001004b86c4cb78549b34bab61a3b1800e23bfeb5b3ec390074041536a7e3cbd97f5f04cf0f857155a8928eaa29ebfd11cfbbad3ba70efea7bda3226c6a8d370a4cd303f714486b6ebc225985a638471e6ef571cc92a4613c00b8fa65d61ccee0cbe5f36330c9a01f4183559f1bef24cc2917c6d913e3a541333a1d05d9bed22b38cb")]
 
 [assembly: WantsToBeBestFriends]
+[assembly: InternalsVisibleTo("Microsoft.ML.CodeGenerator, PublicKey=00240000048000009400000006020000002400005253413100040000010001004b86c4cb78549b34bab61a3b1800e23bfeb5b3ec390074041536a7e3cbd97f5f04cf0f857155a8928eaa29ebfd11cfbbad3ba70efea7bda3226c6a8d370a4cd303f714486b6ebc225985a638471e6ef571cc92a4613c00b8fa65d61ccee0cbe5f36330c9a01f4183559f1bef24cc2917c6d913e3a541333a1d05d9bed22b38cb")]
+[assembly: InternalsVisibleTo("Microsoft.ML.ModelBuilder, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")]

src/Microsoft.ML.AutoML/ColumnInference/ColumnInformationUtil.cs

@@ -47,6 +47,11 @@ internal static class ColumnInformationUtil
                 return ColumnPurpose.Ignore;
             }
 
+            if (columnInfo.LabelCategoricalColumnNames.Contains(columnName))
+            {
+                return ColumnPurpose.LabelFeature;
+            }
+
             return null;
         }
 
@@ -76,6 +81,9 @@ internal static ColumnInformation BuildColumnInfo(IEnumerable<(string name, Colu
                     case ColumnPurpose.NumericFeature:
                         columnInfo.NumericColumnNames.Add(column.name);
                         break;
+                    case ColumnPurpose.LabelFeature:
+                        columnInfo.LabelCategoricalColumnNames.Add(column.name);
+                        break;
                     case ColumnPurpose.TextFeature:
                         columnInfo.TextColumnNames.Add(column.name);
                         break;
@@ -104,6 +112,7 @@ public static IEnumerable<string> GetColumnNames(ColumnInformation columnInforma
             AddStringsToListIfNotNull(columnNames, columnInformation.IgnoredColumnNames);
             AddStringsToListIfNotNull(columnNames, columnInformation.NumericColumnNames);
             AddStringsToListIfNotNull(columnNames, columnInformation.TextColumnNames);
+            AddStringsToListIfNotNull(columnNames, columnInformation.LabelCategoricalColumnNames);
             return columnNames;
         }
 

src/Microsoft.ML.AutoML/ColumnInference/ColumnPurpose.cs

@@ -13,6 +13,7 @@ internal enum ColumnPurpose
         TextFeature = 4,
         Weight = 5,
         ImagePath = 6,
-        SamplingKey = 7
+        SamplingKey = 7,
+        LabelFeature,   // CategoricalFeature that requires ValueToKey converter, better naming?
     }
 }

src/Microsoft.ML.AutoML/Microsoft.ML.AutoML.csproj

@@ -10,6 +10,7 @@
     <ProjectReference Include="..\Microsoft.ML.CpuMath\Microsoft.ML.CpuMath.csproj" />
     <ProjectReference Include="..\Microsoft.ML.LightGbm\Microsoft.ML.LightGbm.csproj" />
     <ProjectReference Include="..\Microsoft.ML.Mkl.Components\Microsoft.ML.Mkl.Components.csproj" />
+    <ProjectReference Include="..\Microsoft.ML.Recommender\Microsoft.ML.Recommender.csproj" />
     <ProjectReference Include="..\Microsoft.ML.Transforms\Microsoft.ML.Transforms.csproj" />
   </ItemGroup>
 

src/Microsoft.ML.AutoML/TaskKind.cs

@@ -9,5 +9,6 @@ internal enum TaskKind
         BinaryClassification,
         MulticlassClassification,
         Regression,
+        Recommendation,
     }
 }

src/Microsoft.ML.AutoML/TrainerExtensions/RecommendationTrainerExtensions.cs

@@ -0,0 +1,37 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Microsoft.ML.Data;
+using Microsoft.ML.Trainers;
+using Microsoft.ML.Trainers.Recommender;
+
+namespace Microsoft.ML.AutoML
+{
+    using ITrainerEsitmator = ITrainerEstimator<ISingleFeaturePredictionTransformer<object>, object>;
+
+    internal class MatrixFactorizationExtension : ITrainerExtension
+    {
+        public ITrainerEsitmator CreateInstance(MLContext mlContext, IEnumerable<SweepableParam> sweepParams, ColumnInformation columnInfo)
+        {
+            // TODO
+            // MatrixFactorizationTrainer.Options should inheriat from ABC TrainerInputBaseWithGroupId
+            var options = TrainerExtensionUtil.CreateOptions<MatrixFactorizationTrainer.Options>(sweepParams, columnInfo.LabelColumnName);
+            options.MatrixColumnIndexColumnName = (string)AutoCatalog.ValuePairs[nameof(options.MatrixColumnIndexColumnName)];
+            options.MatrixRowIndexColumnName = (string)AutoCatalog.ValuePairs[nameof(options.MatrixRowIndexColumnName)];
+            return mlContext.Recommendation().Trainers.MatrixFactorization(options);
+        }
+
+        public PipelineNode CreatePipelineNode(IEnumerable<SweepableParam> sweepParams, ColumnInformation columnInfo)
+        {
+            var property = new Dictionary<string, object>();
+            property.Add(nameof(MatrixFactorizationTrainer.Options.MatrixColumnIndexColumnName), AutoCatalog.ValuePairs[nameof(MatrixFactorizationTrainer.Options.MatrixColumnIndexColumnName)]);
+            property.Add(nameof(MatrixFactorizationTrainer.Options.MatrixRowIndexColumnName), AutoCatalog.ValuePairs[nameof(MatrixFactorizationTrainer.Options.MatrixRowIndexColumnName)]);
+            return TrainerExtensionUtil.BuildPipelineNode(TrainerExtensionCatalog.GetTrainerName(this), sweepParams, columnInfo.LabelColumnName, additionalProperties:property);
+        }
+
+        public IEnumerable<SweepableParam> GetHyperparamSweepRanges()
+        {
+            return SweepableParams.BuildMatrixFactorizationParmas();
+        }
+    }
+}

src/Microsoft.ML.AutoML/TrainerExtensions/SweepableParams.cs

@@ -2,8 +2,10 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
+using System.Collections;
 using System.Collections.Generic;
 using System.Linq;
+using Microsoft.ML.Trainers;
 
 namespace Microsoft.ML.AutoML
 {
@@ -115,6 +117,15 @@ public static IEnumerable<SweepableParam> BuildLightGbmParams()
             };
         }
 
+        public static IEnumerable<SweepableParam> BuildMatrixFactorizationParmas()
+        {
+            return new SweepableParam[]
+            {
+                new SweepableDiscreteParam(nameof(MatrixFactorizationTrainer.Options.NumberOfIterations), new object[] { 10, 20 }),
+                new SweepableFloatParam(nameof(MatrixFactorizationTrainer.Options.LearningRate), 0.025f, 0.04f, numSteps:2),
+                new SweepableDiscreteParam(nameof(MatrixFactorizationTrainer.Options.ApproximationRank), new object[] { 10, 20 }),
+            };
+        }
         public static IEnumerable<SweepableParam> BuildLinearSvmParams()
         {
             return new SweepableParam[] {