Enabling Ranking Cross Validation

src/Microsoft.ML.AutoML/API/ColumnInference.cs

@@ -60,6 +60,7 @@ public sealed class ColumnInformation
 
         /// <summary>
         /// The dataset column to use as a group ID for computation.
+        /// If a SamplingKeyColumnName is provided, then it should be the same as this column.
         /// </summary>
         public string GroupIdColumnName { get; set; }
 

src/Microsoft.ML.AutoML/API/ExperimentBase.cs

@@ -67,11 +67,23 @@ internal ExperimentBase(MLContext context,
         public ExperimentResult<TMetrics> Execute(IDataView trainData, string labelColumnName = DefaultColumnNames.Label,
             string samplingKeyColumn = null, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
         {
-            var columnInformation = new ColumnInformation()
+            ColumnInformation columnInformation;
+            if (_task == TaskKind.Ranking)
             {
-                LabelColumnName = labelColumnName,
-                SamplingKeyColumnName = samplingKeyColumn
-            };
+                columnInformation = new ColumnInformation()
+                {
+                    LabelColumnName = labelColumnName,
+                    GroupIdColumnName = samplingKeyColumn ?? DefaultColumnNames.GroupId
+                };
+            }
+            else
+            {
+                columnInformation = new ColumnInformation()
+                {
+                    LabelColumnName = labelColumnName,
+                    SamplingKeyColumnName = samplingKeyColumn
+                };
+            }
             return Execute(trainData, columnInformation, preFeaturizer, progressHandler);
         }
 
@@ -102,19 +114,28 @@ public ExperimentResult<TMetrics> Execute(IDataView trainData, ColumnInformation
             const int crossValRowCountThreshold = 15000;
 
             var rowCount = DatasetDimensionsUtil.CountRows(trainData, crossValRowCountThreshold);
+            var samplingKeyColumnName = GetSamplingKey(columnInformation?.GroupIdColumnName, columnInformation?.SamplingKeyColumnName);
             if (rowCount < crossValRowCountThreshold)
             {
                 const int numCrossValFolds = 10;
-                var splitResult = SplitUtil.CrossValSplit(Context, trainData, numCrossValFolds, columnInformation?.SamplingKeyColumnName);
+                var splitResult = SplitUtil.CrossValSplit(Context, trainData, numCrossValFolds, samplingKeyColumnName);
                 return ExecuteCrossValSummary(splitResult.trainDatasets, columnInformation, splitResult.validationDatasets, preFeaturizer, progressHandler);
             }
             else
             {
-                var splitResult = SplitUtil.TrainValidateSplit(Context, trainData, columnInformation?.SamplingKeyColumnName);
+                var splitResult = SplitUtil.TrainValidateSplit(Context, trainData, samplingKeyColumnName);
                 return ExecuteTrainValidate(splitResult.trainData, columnInformation, splitResult.validationData, preFeaturizer, progressHandler);
             }
         }
 
+        private string GetSamplingKey(string groupIdColumnName, string samplingKeyColumnName)
+        {
+            UserInputValidationUtil.ValidateSamplingKey(samplingKeyColumnName, groupIdColumnName, _task);
+            if ( _task == TaskKind.Ranking)
+                return groupIdColumnName ?? DefaultColumnNames.GroupId;
+            return samplingKeyColumnName;
+        }
+
         /// <summary>
         /// Executes an AutoML experiment.
         /// </summary>
@@ -194,7 +215,8 @@ public CrossValidationExperimentResult<TMetrics> Execute(IDataView trainData, ui
             IProgress<CrossValidationRunDetail<TMetrics>> progressHandler = null)
         {
             UserInputValidationUtil.ValidateNumberOfCVFoldsArg(numberOfCVFolds);
-            var splitResult = SplitUtil.CrossValSplit(Context, trainData, numberOfCVFolds, columnInformation?.SamplingKeyColumnName);
+            var samplingKeyColumnName = GetSamplingKey(columnInformation?.GroupIdColumnName, columnInformation?.SamplingKeyColumnName);
+            var splitResult = SplitUtil.CrossValSplit(Context, trainData, numberOfCVFolds, samplingKeyColumnName);
             return ExecuteCrossVal(splitResult.trainDatasets, columnInformation, splitResult.validationDatasets, preFeaturizer, progressHandler);
         }
 

src/Microsoft.ML.AutoML/API/RankingExperiment.cs

@@ -34,7 +34,7 @@ public sealed class RankingExperimentSettings : ExperimentSettings
         public ICollection<RankingTrainer> Trainers { get; }
         public RankingExperimentSettings()
         {
-            GroupIdColumnName = "GroupId";
+            GroupIdColumnName = DefaultColumnNames.GroupId;
             OptimizingMetric = RankingMetric.Ndcg;
             Trainers = Enum.GetValues(typeof(RankingTrainer)).OfType<RankingTrainer>().ToList();
         }
@@ -77,7 +77,7 @@ public static class RankingExperimentResultExtensions
         /// <param name="metric">Metric to consider when selecting the best run.</param>
         /// <param name="groupIdColumnName">Name for the GroupId column.</param>
         /// <returns>The best experiment run.</returns>
-        public static RunDetail<RankingMetrics> Best(this IEnumerable<RunDetail<RankingMetrics>> results, RankingMetric metric = RankingMetric.Ndcg, string groupIdColumnName = "GroupId")
+        public static RunDetail<RankingMetrics> Best(this IEnumerable<RunDetail<RankingMetrics>> results, RankingMetric metric = RankingMetric.Ndcg, string groupIdColumnName = DefaultColumnNames.GroupId)
         {
             var metricsAgent = new RankingMetricsAgent(null, metric, groupIdColumnName);
             var isMetricMaximizing = new OptimizingMetricInfo(metric).IsMaximizing;
@@ -91,7 +91,7 @@ public static RunDetail<RankingMetrics> Best(this IEnumerable<RunDetail<RankingM
         /// <param name="metric">Metric to consider when selecting the best run.</param>
         /// <param name="groupIdColumnName">Name for the GroupId column.</param>
         /// <returns>The best experiment run.</returns>
-        public static CrossValidationRunDetail<RankingMetrics> Best(this IEnumerable<CrossValidationRunDetail<RankingMetrics>> results, RankingMetric metric = RankingMetric.Ndcg, string groupIdColumnName = "GroupId")
+        public static CrossValidationRunDetail<RankingMetrics> Best(this IEnumerable<CrossValidationRunDetail<RankingMetrics>> results, RankingMetric metric = RankingMetric.Ndcg, string groupIdColumnName = DefaultColumnNames.GroupId)
         {
             var metricsAgent = new RankingMetricsAgent(null, metric, groupIdColumnName);
             var isMetricMaximizing = new OptimizingMetricInfo(metric).IsMaximizing;

src/Microsoft.ML.AutoML/Utils/UserInputValidationUtil.cs

@@ -57,6 +57,14 @@ public static void ValidateNumberOfCVFoldsArg(uint numberOfCVFolds)
             }
         }
 
+        public static void ValidateSamplingKey(string samplingKeyColumnName, string groupIdColumnName, TaskKind task)
+        {
+            if (task == TaskKind.Ranking && samplingKeyColumnName != null && samplingKeyColumnName != groupIdColumnName)
+            {
+                throw new ArgumentException($"If provided, {nameof(samplingKeyColumnName)} must be the same as {nameof(groupIdColumnName)} for Ranking Experiments", samplingKeyColumnName);
+            }
+        }
+
         private static void ValidateTrainData(IDataView trainData, ColumnInformation columnInformation)
         {
             if (trainData == null)

src/Microsoft.ML.CodeGenerator/Templates/Console/ModelBuilder.cs

@@ -417,7 +417,7 @@ public static void PrintRegressionFoldsAverageMetrics(IEnumerable<TrainCatalogBa
 
         public static void PrintRankingFoldsAverageMetrics(IEnumerable<TrainCatalogBase.CrossValidationResult<RankingMetrics>> crossValidationResults)
         {
-            var max = (crossValidationResults.First().Metrics.NormalizedDiscountedCumulativeGains.Count < 10) ? metrics.NormalizedDiscountedCumulativeGains.Count-1 : 9;
+            var max = (crossValidationResults.First().Metrics.NormalizedDiscountedCumulativeGains.Count < 10) ? crossValidationResults.First().Metrics.NormalizedDiscountedCumulativeGains.Count-1 : 9;
             var NDCG = crossValidationResults.Select(r => r.Metrics.NormalizedDiscountedCumulativeGains[max]);
             var DCG = crossValidationResults.Select(r => r.Metrics.DiscountedCumulativeGains[max]);
             Console.WriteLine($""*************************************************************************************************************"");

src/Microsoft.ML.CodeGenerator/Templates/Console/ModelBuilder.tt

@@ -336,7 +336,7 @@ else{#>
 
         public static void PrintRankingFoldsAverageMetrics(IEnumerable<TrainCatalogBase.CrossValidationResult<RankingMetrics>> crossValidationResults)
         {
-            var max = (crossValidationResults.First().Metrics.NormalizedDiscountedCumulativeGains.Count < 10) ? metrics.NormalizedDiscountedCumulativeGains.Count-1 : 9;
+            var max = (crossValidationResults.First().Metrics.NormalizedDiscountedCumulativeGains.Count < 10) ? crossValidationResults.First().Metrics.NormalizedDiscountedCumulativeGains.Count-1 : 9;
             var NDCG = crossValidationResults.Select(r => r.Metrics.NormalizedDiscountedCumulativeGains[max]);
             var DCG = crossValidationResults.Select(r => r.Metrics.DiscountedCumulativeGains[max]);
             Console.WriteLine($"*************************************************************************************************************");

src/Microsoft.ML.Data/DataLoadSave/DataOperationsCatalog.cs

@@ -398,6 +398,7 @@ public IDataView TakeRows(IDataView input, long count)
         /// <param name="testFraction">The fraction of data to go into the test set.</param>
         /// <param name="samplingKeyColumnName">Name of a column to use for grouping rows. If two examples share the same value of the <paramref name="samplingKeyColumnName"/>,
         /// they are guaranteed to appear in the same subset (train or test). This can be used to ensure no label leakage from the train to the test set.
+        /// Note that when performing a Ranking Experiment, the <paramref name="samplingKeyColumnName"/> must be the GroupId column.
         /// If <see langword="null"/> no row grouping will be performed.</param>
         /// <param name="seed">Seed for the random number generator used to select rows for the train-test split.</param>
         /// <example>
@@ -444,6 +445,7 @@ public TrainTestData TrainTestSplit(IDataView data, double testFraction = 0.1, s
         /// <param name="numberOfFolds">Number of cross-validation folds.</param>
         /// <param name="samplingKeyColumnName">Name of a column to use for grouping rows. If two examples share the same value of the <paramref name="samplingKeyColumnName"/>,
         /// they are guaranteed to appear in the same subset (train or test). This can be used to ensure no label leakage from the train to the test set.
+        /// Note that when performing a Ranking Experiment, the <paramref name="samplingKeyColumnName"/> must be the GroupId column.
         /// If <see langword="null"/> no row grouping will be performed.</param>
         /// <param name="seed">Seed for the random number generator used to select rows for cross-validation folds.</param>
         /// <example>

src/Microsoft.ML.Data/Evaluators/RankingEvaluator.cs

@@ -63,7 +63,7 @@ internal sealed class RankingEvaluator : EvaluatorBase<RankingEvaluator.Aggregat
         /// </value>
         public const string GroupSummary = "GroupSummary";
 
-        private const string GroupId = "GroupId";
+        private const string GroupId = DefaultColumnNames.GroupId;
 
         private readonly int _truncationLevel;
         private readonly bool _groupSummary;

src/Microsoft.ML.Data/TrainCatalog.cs

@@ -674,6 +674,31 @@ public RankingMetrics Evaluate(IDataView data,
             var eval = new RankingEvaluator(Environment, options ?? new RankingEvaluatorOptions() { });
             return eval.Evaluate(data, labelColumnName, rowGroupColumnName, scoreColumnName);
         }
+
+        /// <summary>
+        /// Run cross-validation over <paramref name="numberOfFolds"/> folds of <paramref name="data"/>, by fitting <paramref name="estimator"/>,
+        /// and respecting <paramref name="rowGroupColumnName"/>if provided.
+        /// Then evaluate each sub-model against <paramref name="labelColumnName"/> and return metrics.
+        /// </summary>
+        /// <param name="data">The data to run cross-validation on.</param>
+        /// <param name="estimator">The estimator to fit.</param>
+        /// <param name="numberOfFolds">Number of cross-validation folds.</param>
+        /// <param name="labelColumnName">The label column (for evaluation).</param>
+        /// <param name="rowGroupColumnName">The name of the groupId column in <paramref name="data"/>, which is used to group rows.
+        /// While for other crossvalidation methods this column is called samplingKeyColumnName, ranking requires
+        /// this column to be <paramref name="rowGroupColumnName"/>.
+        /// If <see langword="null"/> no row grouping will be performed. </param>
+        /// <param name="seed">  Seed for the random number generator used to select rows for cross-validation folds.</param>
+        /// <returns>Per-fold results: metrics, models, scored datasets.</returns>
+        public IReadOnlyList<CrossValidationResult<RankingMetrics>> CrossValidate(
+            IDataView data, IEstimator<ITransformer> estimator, int numberOfFolds = 5, string labelColumnName = DefaultColumnNames.Label,
+            string rowGroupColumnName = DefaultColumnNames.GroupId, int ? seed = null)
+        {
+            Environment.CheckNonEmpty(labelColumnName, nameof(labelColumnName));
+            var result = CrossValidateTrain(data, estimator, numberOfFolds, rowGroupColumnName, seed);
+            return result.Select(x => new CrossValidationResult<RankingMetrics>(x.Model,
+                Evaluate(x.Scores, labelColumnName, rowGroupColumnName), x.Scores, x.Fold)).ToArray();
+        }
     }
 
     /// <summary>

src/Microsoft.ML.LightGbm/LightGbmRankingTrainer.cs

@@ -179,7 +179,7 @@ internal LightGbmRankingTrainer(IHostEnvironment env, Options options)
         /// <param name="env">The private instance of <see cref="IHostEnvironment"/>.</param>
         /// <param name="labelColumnName">The name of the label column.</param>
         /// <param name="featureColumnName">The name of the feature column.</param>
-        /// <param name="rowGroupdColumnName">The name of the column containing the group ID. </param>
+        /// <param name="rowGroupIdColumnName">The name of the column containing the group ID. </param>
         /// <param name="weightsColumnName">The name of the optional column containing the initial weights.</param>
         /// <param name="numberOfLeaves">The number of leaves to use.</param>
         /// <param name="learningRate">The learning rate.</param>
@@ -188,7 +188,7 @@ internal LightGbmRankingTrainer(IHostEnvironment env, Options options)
         internal LightGbmRankingTrainer(IHostEnvironment env,
             string labelColumnName = DefaultColumnNames.Label,
             string featureColumnName = DefaultColumnNames.Features,
-            string rowGroupdColumnName = DefaultColumnNames.GroupId,
+            string rowGroupIdColumnName = DefaultColumnNames.GroupId,
             string weightsColumnName = null,
             int? numberOfLeaves = null,
             int? minimumExampleCountPerLeaf = null,
@@ -200,14 +200,14 @@ internal LightGbmRankingTrainer(IHostEnvironment env,
                       LabelColumnName = labelColumnName,
                       FeatureColumnName = featureColumnName,
                       ExampleWeightColumnName = weightsColumnName,
-                      RowGroupColumnName = rowGroupdColumnName,
+                      RowGroupColumnName = rowGroupIdColumnName,
                       NumberOfLeaves = numberOfLeaves,
                       MinimumExampleCountPerLeaf = minimumExampleCountPerLeaf,
                       LearningRate = learningRate,
                       NumberOfIterations = numberOfIterations
                   })
         {
-            Host.CheckNonEmpty(rowGroupdColumnName, nameof(rowGroupdColumnName));
+            Host.CheckNonEmpty(rowGroupIdColumnName, nameof(rowGroupIdColumnName));
         }
 
         private protected override void CheckDataValid(IChannel ch, RoleMappedData data)

test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs

@@ -7,6 +7,7 @@
 using Microsoft.ML.TestFramework;
 using Microsoft.ML.TestFramework.Attributes;
 using Microsoft.ML.TestFrameworkCommon;
+using Microsoft.ML.Trainers.LightGbm;
 using Xunit;
 using Xunit.Abstractions;
 using static Microsoft.ML.DataOperationsCatalog;
@@ -156,6 +157,40 @@ public void AutoFitRankingTest()
                 Assert.True(col.Name == expectedOutputNames[col.Index]);
         }
 
+        [LightGBMFact]
+        public void AutoFitRankingCVTest()
+        {
+            string labelColumnName = "Label";
+            string groupIdColumnName = "GroupIdCustom";
+            string featuresColumnVectorNameA = "FeatureVectorA";
+            string featuresColumnVectorNameB = "FeatureVectorB";
+            uint numFolds = 3;
+
+            var mlContext = new MLContext(1);
+            var reader = new TextLoader(mlContext, GetLoaderArgsRank(labelColumnName, groupIdColumnName,
+                featuresColumnVectorNameA, featuresColumnVectorNameB));
+            var trainDataView = reader.Load(new MultiFileSource(DatasetUtil.GetMLSRDataset()));
+
+            CrossValidationExperimentResult<RankingMetrics> experimentResult = mlContext.Auto()
+                .CreateRankingExperiment(new RankingExperimentSettings() { GroupIdColumnName = groupIdColumnName, MaxExperimentTimeInSeconds = 5 })
+                .Execute(trainDataView, numFolds,
+                    new ColumnInformation()
+                    {
+                        LabelColumnName = labelColumnName,
+                        GroupIdColumnName = groupIdColumnName
+                    });
+
+            CrossValidationRunDetail<RankingMetrics> bestRun = experimentResult.BestRun;
+            Assert.True(experimentResult.RunDetails.Count() > 0);
+            var enumerator = bestRun.Results.GetEnumerator();
+            while (enumerator.MoveNext())
+            {
+                var model = enumerator.Current;
+                Assert.True(model.ValidationMetrics.NormalizedDiscountedCumulativeGains.Max() > .4);
+                Assert.True(model.ValidationMetrics.DiscountedCumulativeGains.Max() > 19);   
+            }
+        }
+
         [Fact]
         public void AutoFitRecommendationTest()
         {

test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.ConsoleAppModelBuilderCSFileContentRankingTest.approved.txt

@@ -11,7 +11,6 @@ using System.Linq;
 using Microsoft.ML;
 using Microsoft.ML.Data;
 using TestNamespace.Model;
-using Microsoft.ML.Trainers.LightGbm;
 
 namespace TestNamespace.ConsoleApp
 {
@@ -58,7 +57,7 @@ namespace TestNamespace.ConsoleApp
             // Data process configuration with pipeline data transformations 
             var dataProcessPipeline = mlContext.Transforms.Conversion.Hash("GroupId", "GroupId");
             // Set the training algorithm 
-            var trainer = mlContext.Ranking.Trainers.LightGbm(new LightGbmRankingTrainer.Options() { rowGroupColumnName = "GroupId", LabelColumnName = "Label", FeatureColumnName = "Features" });
+            var trainer = mlContext.Ranking.Trainers.LightGbm(rowGroupColumnName: "GroupId", labelColumnName: "Label", featureColumnName: "Features");
 
             var trainingPipeline = dataProcessPipeline.Append(trainer);
 
@@ -115,7 +114,7 @@ namespace TestNamespace.ConsoleApp
 
         public static void PrintRankingFoldsAverageMetrics(IEnumerable<TrainCatalogBase.CrossValidationResult<RankingMetrics>> crossValidationResults)
         {
-            var max = (crossValidationResults.First().Metrics.NormalizedDiscountedCumulativeGains.Count < 10) ? metrics.NormalizedDiscountedCumulativeGains.Count - 1 : 9;
+            var max = (crossValidationResults.First().Metrics.NormalizedDiscountedCumulativeGains.Count < 10) ? crossValidationResults.First().Metrics.NormalizedDiscountedCumulativeGains.Count - 1 : 9;
             var NDCG = crossValidationResults.Select(r => r.Metrics.NormalizedDiscountedCumulativeGains[max]);
             var DCG = crossValidationResults.Select(r => r.Metrics.DiscountedCumulativeGains[max]);
             Console.WriteLine($"*************************************************************************************************************");