adding cross validation for ranking

src/Microsoft.ML.CodeGenerator/Templates/Console/ModelBuilder.cs

@@ -417,7 +417,7 @@ public static void PrintRegressionFoldsAverageMetrics(IEnumerable<TrainCatalogBa
 
         public static void PrintRankingFoldsAverageMetrics(IEnumerable<TrainCatalogBase.CrossValidationResult<RankingMetrics>> crossValidationResults)
         {
-            var max = (crossValidationResults.First().Metrics.NormalizedDiscountedCumulativeGains.Count < 10) ? metrics.NormalizedDiscountedCumulativeGains.Count-1 : 9;
+            var max = (crossValidationResults.First().Metrics.NormalizedDiscountedCumulativeGains.Count < 10) ? crossValidationResults.First().Metrics.NormalizedDiscountedCumulativeGains.Count-1 : 9;
             var NDCG = crossValidationResults.Select(r => r.Metrics.NormalizedDiscountedCumulativeGains[max]);
             var DCG = crossValidationResults.Select(r => r.Metrics.DiscountedCumulativeGains[max]);
             Console.WriteLine($""*************************************************************************************************************"");

src/Microsoft.ML.CodeGenerator/Templates/Console/ModelBuilder.tt

@@ -336,7 +336,7 @@ else{#>
 
         public static void PrintRankingFoldsAverageMetrics(IEnumerable<TrainCatalogBase.CrossValidationResult<RankingMetrics>> crossValidationResults)
         {
-            var max = (crossValidationResults.First().Metrics.NormalizedDiscountedCumulativeGains.Count < 10) ? metrics.NormalizedDiscountedCumulativeGains.Count-1 : 9;
+            var max = (crossValidationResults.First().Metrics.NormalizedDiscountedCumulativeGains.Count < 10) ? crossValidationResults.First().Metrics.NormalizedDiscountedCumulativeGains.Count-1 : 9;
             var NDCG = crossValidationResults.Select(r => r.Metrics.NormalizedDiscountedCumulativeGains[max]);
             var DCG = crossValidationResults.Select(r => r.Metrics.DiscountedCumulativeGains[max]);
             Console.WriteLine($"*************************************************************************************************************");

src/Microsoft.ML.Data/TrainCatalog.cs

@@ -674,6 +674,31 @@ public RankingMetrics Evaluate(IDataView data,
             var eval = new RankingEvaluator(Environment, options ?? new RankingEvaluatorOptions() { });
             return eval.Evaluate(data, labelColumnName, rowGroupColumnName, scoreColumnName);
         }
+
+        /// <summary>
+        /// Run cross-validation over <paramref name="numberOfFolds"/> folds of <paramref name="data"/>, by fitting <paramref name="estimator"/>,
+        /// and respecting <paramref name="samplingKeyColumnName"/> if provided.
+        /// Then evaluate each sub-model against <paramref name="labelColumnName"/> and return metrics.
+        /// </summary>
+        /// <param name="data">The data to run cross-validation on.</param>
+        /// <param name="estimator">The estimator to fit.</param>
+        /// <param name="numberOfFolds">Number of cross-validation folds.</param>
+        /// <param name="labelColumnName">The label column (for evaluation).</param>
+        /// <param name="samplingKeyColumnName">Name of a column to use for grouping rows. If two examples share the same value of the <paramref name="samplingKeyColumnName"/>,
+        /// they are guaranteed to appear in the same subset (train or test). This can be used to ensure no label leakage from the train to the test set.
+        /// If <see langword="null"/> no row grouping will be performed.</param>
+        /// <param name="rowGroupColumnName">The name of the groupId column in <paramref name="data"/>.</param>
+        /// <param name="seed">Seed for the random number generator used to select rows for cross-validation folds.</param>
+        /// <returns>Per-fold results: metrics, models, scored datasets.</returns>
+        public IReadOnlyList<CrossValidationResult<RankingMetrics>> CrossValidate(
+            IDataView data, IEstimator<ITransformer> estimator, int numberOfFolds = 5, string labelColumnName = DefaultColumnNames.Label,
+            string samplingKeyColumnName = DefaultColumnNames.GroupId, string rowGroupColumnName = DefaultColumnNames.GroupId, int ? seed = null)
+        {
+            Environment.CheckNonEmpty(labelColumnName, nameof(labelColumnName));
+            var result = CrossValidateTrain(data, estimator, numberOfFolds, samplingKeyColumnName, seed);
+            return result.Select(x => new CrossValidationResult<RankingMetrics>(x.Model,
+                Evaluate(x.Scores, labelColumnName, rowGroupColumnName), x.Scores, x.Fold)).ToArray();
+        }
     }
 
     /// <summary>

src/Microsoft.ML.LightGbm/LightGbmRankingTrainer.cs

@@ -179,7 +179,7 @@ internal LightGbmRankingTrainer(IHostEnvironment env, Options options)
         /// <param name="env">The private instance of <see cref="IHostEnvironment"/>.</param>
         /// <param name="labelColumnName">The name of the label column.</param>
         /// <param name="featureColumnName">The name of the feature column.</param>
-        /// <param name="rowGroupdColumnName">The name of the column containing the group ID. </param>
+        /// <param name="rowGroupIdColumnName">The name of the column containing the group ID. </param>
         /// <param name="weightsColumnName">The name of the optional column containing the initial weights.</param>
         /// <param name="numberOfLeaves">The number of leaves to use.</param>
         /// <param name="learningRate">The learning rate.</param>
@@ -188,7 +188,7 @@ internal LightGbmRankingTrainer(IHostEnvironment env, Options options)
         internal LightGbmRankingTrainer(IHostEnvironment env,
             string labelColumnName = DefaultColumnNames.Label,
             string featureColumnName = DefaultColumnNames.Features,
-            string rowGroupdColumnName = DefaultColumnNames.GroupId,
+            string rowGroupIdColumnName = DefaultColumnNames.GroupId,
             string weightsColumnName = null,
             int? numberOfLeaves = null,
             int? minimumExampleCountPerLeaf = null,
@@ -200,14 +200,14 @@ internal LightGbmRankingTrainer(IHostEnvironment env,
                       LabelColumnName = labelColumnName,
                       FeatureColumnName = featureColumnName,
                       ExampleWeightColumnName = weightsColumnName,
-                      RowGroupColumnName = rowGroupdColumnName,
+                      RowGroupColumnName = rowGroupIdColumnName,
                       NumberOfLeaves = numberOfLeaves,
                       MinimumExampleCountPerLeaf = minimumExampleCountPerLeaf,
                       LearningRate = learningRate,
                       NumberOfIterations = numberOfIterations
                   })
         {
-            Host.CheckNonEmpty(rowGroupdColumnName, nameof(rowGroupdColumnName));
+            Host.CheckNonEmpty(rowGroupIdColumnName, nameof(rowGroupIdColumnName));
         }
 
         private protected override void CheckDataValid(IChannel ch, RoleMappedData data)

test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs

@@ -8,6 +8,7 @@
 using Microsoft.ML.TestFramework.Attributes;
 using Microsoft.ML.TestFrameworkCommon;
 using Xunit;
+using Microsoft.ML.Trainers.LightGbm;
 using Xunit.Abstractions;
 using static Microsoft.ML.DataOperationsCatalog;
 
@@ -156,6 +157,31 @@ public void AutoFitRankingTest()
                 Assert.True(col.Name == expectedOutputNames[col.Index]);
         }
 
+        [LightGBMFact]
+        public void AutoFitRankingCVTest()
+        {
+            string labelColumnName = "Label";
+            string groupIdColumnName = "GroupId";
+            string featuresColumnVectorNameA = "FeatureVectorA";
+            string featuresColumnVectorNameB = "FeatureVectorB";
+            int numFolds = 3;
+
+            var mlContext = new MLContext(1);
+            var dataProcessPipeline = mlContext.Transforms.Concatenate("Features", new[] { "FeatureVectorA", "FeatureVectorB" }).Append(
+                mlContext.Transforms.Conversion.Hash("GroupId", "GroupId"));
+
+            var trainer = mlContext.Ranking.Trainers.LightGbm(new LightGbmRankingTrainer.Options() { RowGroupColumnName = "GroupId", LabelColumnName = "Label", FeatureColumnName = "Features" });
+            var reader = new TextLoader(mlContext, GetLoaderArgsRank(labelColumnName, groupIdColumnName, featuresColumnVectorNameA, featuresColumnVectorNameB));
+            var trainDataView = reader.Load(new MultiFileSource(DatasetUtil.GetMLSRDataset()));
+            var trainingPipeline = dataProcessPipeline.Append(trainer);
+            var result = mlContext.Ranking.CrossValidate(trainDataView, trainingPipeline, numberOfFolds: numFolds);
+            for (int i = 0; i < numFolds; i++)
+            {
+                Assert.True(result[i].Metrics.NormalizedDiscountedCumulativeGains.Max() > .4);
+                Assert.True(result[i].Metrics.DiscountedCumulativeGains.Max() > 16);
+            }
+        }
+
         [Fact]
         public void AutoFitRecommendationTest()
         {

test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.ConsoleAppModelBuilderCSFileContentRankingTest.approved.txt

@@ -11,7 +11,6 @@ using System.Linq;
 using Microsoft.ML;
 using Microsoft.ML.Data;
 using TestNamespace.Model;
-using Microsoft.ML.Trainers.LightGbm;
 
 namespace TestNamespace.ConsoleApp
 {
@@ -58,7 +57,7 @@ namespace TestNamespace.ConsoleApp
             // Data process configuration with pipeline data transformations 
             var dataProcessPipeline = mlContext.Transforms.Conversion.Hash("GroupId", "GroupId");
             // Set the training algorithm 
-            var trainer = mlContext.Ranking.Trainers.LightGbm(new LightGbmRankingTrainer.Options() { rowGroupColumnName = "GroupId", LabelColumnName = "Label", FeatureColumnName = "Features" });
+            var trainer = mlContext.Ranking.Trainers.LightGbm(rowGroupColumnName: "GroupId", labelColumnName: "Label", featureColumnName: "Features");
 
             var trainingPipeline = dataProcessPipeline.Append(trainer);
 
@@ -115,7 +114,7 @@ namespace TestNamespace.ConsoleApp
 
         public static void PrintRankingFoldsAverageMetrics(IEnumerable<TrainCatalogBase.CrossValidationResult<RankingMetrics>> crossValidationResults)
         {
-            var max = (crossValidationResults.First().Metrics.NormalizedDiscountedCumulativeGains.Count < 10) ? metrics.NormalizedDiscountedCumulativeGains.Count - 1 : 9;
+            var max = (crossValidationResults.First().Metrics.NormalizedDiscountedCumulativeGains.Count < 10) ? crossValidationResults.First().Metrics.NormalizedDiscountedCumulativeGains.Count - 1 : 9;
             var NDCG = crossValidationResults.Select(r => r.Metrics.NormalizedDiscountedCumulativeGains[max]);
             var DCG = crossValidationResults.Select(r => r.Metrics.DiscountedCumulativeGains[max]);
             Console.WriteLine($"*************************************************************************************************************");

test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.cs

@@ -623,10 +623,9 @@ private CodeGenerator PrepareForRecommendationTask()
         {
             if (_mockedPipeline == null)
             {
-                MLContext context = new MLContext();
                 var hyperParam = new Dictionary<string, object>()
                 {
-                    {"rowGroupColumnName","GroupId" },
+                    {"RowGroupColumnName","GroupId" },
                     {"LabelColumnName","Label" },
                 };
                 var hashPipelineNode = new PipelineNode(nameof(EstimatorName.Hashing), PipelineNodeType.Transform, "GroupId", "GroupId");