CV test

Author: Lynx1820

src/Microsoft.ML.AutoML/API/ColumnInference.cs

@@ -118,6 +118,7 @@ public sealed class ColumnInformation
         public ColumnInformation()
         {
             LabelColumnName = DefaultColumnNames.Label;
+            GroupIdColumnName = DefaultColumnNames.GroupId;
             CategoricalColumnNames = new Collection<string>();
             NumericColumnNames = new Collection<string>();
             TextColumnNames = new Collection<string>();

src/Microsoft.ML.AutoML/API/ExperimentBase.cs

@@ -194,7 +194,8 @@ public CrossValidationExperimentResult<TMetrics> Execute(IDataView trainData, ui
             IProgress<CrossValidationRunDetail<TMetrics>> progressHandler = null)
         {
             UserInputValidationUtil.ValidateNumberOfCVFoldsArg(numberOfCVFolds);
-            var splitResult = SplitUtil.CrossValSplit(Context, trainData, numberOfCVFolds, columnInformation?.SamplingKeyColumnName);
+            UserInputValidationUtil.ValidateSamplingKey(columnInformation?.SamplingKeyColumnName, columnInformation?.GroupIdColumnName, _task);
+            var splitResult = SplitUtil.CrossValSplit(Context, trainData, numberOfCVFolds, columnInformation?.GroupIdColumnName);
             return ExecuteCrossVal(splitResult.trainDatasets, columnInformation, splitResult.validationDatasets, preFeaturizer, progressHandler);
         }
 

src/Microsoft.ML.AutoML/Utils/UserInputValidationUtil.cs

@@ -57,6 +57,14 @@ public static void ValidateNumberOfCVFoldsArg(uint numberOfCVFolds)
             }
         }
 
+        public static void ValidateSamplingKey(string samplingKeyColumnName, string groupIdColumnName, TaskKind task)
+        {
+            if (task == TaskKind.Ranking && samplingKeyColumnName != null && samplingKeyColumnName != groupIdColumnName)
+            {
+                throw new ArgumentException($"{nameof(samplingKeyColumnName)} must be the same as {nameof(groupIdColumnName)}", samplingKeyColumnName);
+            }
+        }
+
         private static void ValidateTrainData(IDataView trainData, ColumnInformation columnInformation)
         {
             if (trainData == null)

test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs

@@ -7,8 +7,8 @@
 using Microsoft.ML.TestFramework;
 using Microsoft.ML.TestFramework.Attributes;
 using Microsoft.ML.TestFrameworkCommon;
-using Xunit;
 using Microsoft.ML.Trainers.LightGbm;
+using Xunit;
 using Xunit.Abstractions;
 using static Microsoft.ML.DataOperationsCatalog;
 
@@ -182,6 +182,40 @@ public void AutoFitRankingCVTest()
             }
         }
 
+        [LightGBMFact]
+        public void AutoFitRankingCV2Test()
+        {
+            string labelColumnName = "Label";
+            string groupIdColumnName = "GroupId";
+            string featuresColumnVectorNameA = "FeatureVectorA";
+            string featuresColumnVectorNameB = "FeatureVectorB";
+            uint numFolds = 3;
+
+            var mlContext = new MLContext(1);
+            var reader = new TextLoader(mlContext, GetLoaderArgsRank(labelColumnName, groupIdColumnName,
+                featuresColumnVectorNameA, featuresColumnVectorNameB));
+            var trainDataView = reader.Load(new MultiFileSource(DatasetUtil.GetMLSRDataset()));
+
+            CrossValidationExperimentResult<RankingMetrics> experimentResult = mlContext.Auto()
+                .CreateRankingExperiment(5)
+                .Execute(trainDataView, numFolds,
+                    new ColumnInformation()
+                    {
+                        LabelColumnName = labelColumnName,
+                        GroupIdColumnName = groupIdColumnName
+                    });
+
+            CrossValidationRunDetail<RankingMetrics> bestRun = experimentResult.BestRun;
+            Assert.True(experimentResult.RunDetails.Count() > 0);
+            var enumerator = bestRun.Results.GetEnumerator();
+            while (enumerator.MoveNext())
+            {
+                var model = enumerator.Current;
+                Assert.True(model.ValidationMetrics.NormalizedDiscountedCumulativeGains.Max() > .4);
+                Assert.True(model.ValidationMetrics.DiscountedCumulativeGains.Max() > 19);   
+            }
+        }
+
         [Fact]
         public void AutoFitRecommendationTest()
         {