Change default EvaluationMetric for LightGbm trainers to conform to default metric in standalone LightGbm (#3859)

najeeb-kazmi · codemzs · commit 3a35a82b4bbb · 2019-07-01T06:02:35.000-07:00
diff --git a/src/Microsoft.ML.LightGbm/LightGbmBinaryTrainer.cs b/src/Microsoft.ML.LightGbm/LightGbmBinaryTrainer.cs
@@ -162,7 +162,7 @@ public enum EvaluateMetricType
             [Argument(ArgumentType.AtMostOnce,
                 HelpText = "Evaluation metrics.",
                 ShortName = "em")]
-            public EvaluateMetricType EvaluationMetric = EvaluateMetricType.Logloss;
+            public EvaluateMetricType EvaluationMetric = EvaluateMetricType.Default;
 
             static Options()
             {
diff --git a/src/Microsoft.ML.LightGbm/LightGbmMulticlassTrainer.cs b/src/Microsoft.ML.LightGbm/LightGbmMulticlassTrainer.cs
@@ -95,7 +95,7 @@ public enum EvaluateMetricType
             [Argument(ArgumentType.AtMostOnce,
                 HelpText = "Evaluation metrics.",
                 ShortName = "em")]
-            public EvaluateMetricType EvaluationMetric = EvaluateMetricType.Error;
+            public EvaluateMetricType EvaluationMetric = EvaluateMetricType.Default;
 
             static Options()
             {
diff --git a/src/Microsoft.ML.LightGbm/LightGbmRankingTrainer.cs b/src/Microsoft.ML.LightGbm/LightGbmRankingTrainer.cs
@@ -143,7 +143,7 @@ public enum EvaluateMetricType
             [Argument(ArgumentType.AtMostOnce,
                 HelpText = "Evaluation metrics.",
                 ShortName = "em")]
-            public EvaluateMetricType EvaluationMetric = EvaluateMetricType.NormalizedDiscountedCumulativeGain;
+            public EvaluateMetricType EvaluationMetric = EvaluateMetricType.Default;
 
             static Options()
             {
diff --git a/src/Microsoft.ML.LightGbm/LightGbmRegressionTrainer.cs b/src/Microsoft.ML.LightGbm/LightGbmRegressionTrainer.cs
@@ -133,7 +133,7 @@ public enum EvaluateMetricType
             [Argument(ArgumentType.AtMostOnce,
                 HelpText = "Evaluation metrics.",
                 ShortName = "em")]
-            public EvaluateMetricType EvaluationMetric = EvaluateMetricType.RootMeanSquaredError;
+            public EvaluateMetricType EvaluationMetric = EvaluateMetricType.Default;
 
             static Options()
             {
diff --git a/test/BaselineOutput/Common/EntryPoints/core_manifest.json b/test/BaselineOutput/Common/EntryPoints/core_manifest.json
@@ -11535,7 +11535,7 @@
           "Required": false,
           "SortOrder": 150.0,
           "IsNullable": false,
-          "Default": "Logloss"
+          "Default": "Default"
         },
         {
           "Name": "MaximumBinCountPerFeature",
@@ -12020,7 +12020,7 @@
           "Required": false,
           "SortOrder": 150.0,
           "IsNullable": false,
-          "Default": "Error"
+          "Default": "Default"
         },
         {
           "Name": "MaximumBinCountPerFeature",
@@ -12517,7 +12517,7 @@
           "Required": false,
           "SortOrder": 150.0,
           "IsNullable": false,
-          "Default": "NormalizedDiscountedCumulativeGain"
+          "Default": "Default"
         },
         {
           "Name": "MaximumBinCountPerFeature",
@@ -12975,7 +12975,7 @@
           "Required": false,
           "SortOrder": 150.0,
           "IsNullable": false,
-          "Default": "RootMeanSquaredError"
+          "Default": "Default"
         },
         {
           "Name": "MaximumBinCountPerFeature",
diff --git a/test/BaselineOutput/Common/LightGBMR/LightGBMReg-CV-generatedRegressionDataset-out.txt b/test/BaselineOutput/Common/LightGBMR/LightGBMReg-CV-generatedRegressionDataset-out.txt
@@ -35,10 +35,10 @@ Virtual memory usage(MB): %Number%
 [1] 'Loading data for LightGBM' started.
 [1] 'Loading data for LightGBM' finished in %Time%.
 [2] 'Training with LightGBM' started.
-[2] (%Time%)	Iteration: 50	Training-rmse: 6.09160118577349
+[2] (%Time%)	Iteration: 50	Training-: 37.107605006517
 [2] 'Training with LightGBM' finished in %Time%.
 [3] 'Loading data for LightGBM #2' started.
 [3] 'Loading data for LightGBM #2' finished in %Time%.
 [4] 'Training with LightGBM #2' started.
-[4] (%Time%)	Iteration: 50	Training-rmse: 5.26343689176522
+[4] (%Time%)	Iteration: 50	Training-: 27.7037679135951
 [4] 'Training with LightGBM #2' finished in %Time%.
diff --git a/test/BaselineOutput/Common/LightGBMR/LightGBMReg-TrainTest-generatedRegressionDataset-out.txt b/test/BaselineOutput/Common/LightGBMR/LightGBMReg-TrainTest-generatedRegressionDataset-out.txt
@@ -26,7 +26,7 @@ Virtual memory usage(MB): %Number%
 [1] 'Loading data for LightGBM' started.
 [1] 'Loading data for LightGBM' finished in %Time%.
 [2] 'Training with LightGBM' started.
-[2] (%Time%)	Iteration: 50	Training-rmse: 5.10533343749577
+[2] (%Time%)	Iteration: 50	Training-: 26.0644295080124
 [2] 'Training with LightGBM' finished in %Time%.
 [3] 'Saving model' started.
 [3] 'Saving model' finished in %Time%.
diff --git a/test/BaselineOutput/Common/LightGBMR/LightGBMRegRmse-CV-generatedRegressionDataset.RMSE-rp.txt b/test/BaselineOutput/Common/LightGBMR/LightGBMRegRmse-CV-generatedRegressionDataset.RMSE-rp.txt
@@ -1,4 +1,4 @@
 LightGBMR
-L1(avg)	L2(avg)	RMS(avg)	Loss-fn(avg)	R Squared	/iter	/lr	/nl	/mil	/v	/nt	Learner Name	Train Dataset	Test Dataset	Results File	Run Time	Physical Memory	Virtual Memory	Command Line	Settings	
-26.59978	1393.326	37.32081	1393.326	0.923402	50	0.2	20	10	+	1	LightGBMR	%Data%		%Output%	99	0	0	maml.exe CV tr=LightGBMR{nt=1 iter=50 em=RootMeanSquaredError v=+ lr=0.2 mil=10 nl=20} threads=- dout=%Output% loader=Text{col=Label:R4:11 col=Features:R4:0-10 sep=; header+} data=%Data% seed=1	/iter:50;/lr:0.2;/nl:20;/mil:10;/v:+;/nt:1	
+L1(avg)	L2(avg)	RMS(avg)	Loss-fn(avg)	R Squared	/em	/iter	/lr	/nl	/mil	/v	/nt	Learner Name	Train Dataset	Test Dataset	Results File	Run Time	Physical Memory	Virtual Memory	Command Line	Settings	
+26.59978	1393.326	37.32081	1393.326	0.923402	RootMeanSquaredError	50	0.2	20	10	+	1	LightGBMR	%Data%		%Output%	99	0	0	maml.exe CV tr=LightGBMR{nt=1 iter=50 em=RootMeanSquaredError v=+ lr=0.2 mil=10 nl=20} threads=- dout=%Output% loader=Text{col=Label:R4:11 col=Features:R4:0-10 sep=; header+} data=%Data% seed=1	/em:RootMeanSquaredError;/iter:50;/lr:0.2;/nl:20;/mil:10;/v:+;/nt:1	
 
diff --git a/test/BaselineOutput/Common/LightGBMR/LightGBMRegRmse-TrainTest-generatedRegressionDataset.RMSE-rp.txt b/test/BaselineOutput/Common/LightGBMR/LightGBMRegRmse-TrainTest-generatedRegressionDataset.RMSE-rp.txt
@@ -1,4 +1,4 @@
 LightGBMR
-L1(avg)	L2(avg)	RMS(avg)	Loss-fn(avg)	R Squared	/iter	/lr	/nl	/mil	/v	/nt	Learner Name	Train Dataset	Test Dataset	Results File	Run Time	Physical Memory	Virtual Memory	Command Line	Settings	
-3.428896	25.23601	5.023546	25.23601	0.998616	50	0.2	20	10	+	1	LightGBMR	%Data%	%Data%	%Output%	99	0	0	maml.exe TrainTest test=%Data% tr=LightGBMR{nt=1 iter=50 em=RootMeanSquaredError v=+ lr=0.2 mil=10 nl=20} dout=%Output% loader=Text{col=Label:R4:11 col=Features:R4:0-10 sep=; header+} data=%Data% out=%Output% seed=1	/iter:50;/lr:0.2;/nl:20;/mil:10;/v:+;/nt:1	
+L1(avg)	L2(avg)	RMS(avg)	Loss-fn(avg)	R Squared	/em	/iter	/lr	/nl	/mil	/v	/nt	Learner Name	Train Dataset	Test Dataset	Results File	Run Time	Physical Memory	Virtual Memory	Command Line	Settings	
+3.428896	25.23601	5.023546	25.23601	0.998616	RootMeanSquaredError	50	0.2	20	10	+	1	LightGBMR	%Data%	%Data%	%Output%	99	0	0	maml.exe TrainTest test=%Data% tr=LightGBMR{nt=1 iter=50 em=RootMeanSquaredError v=+ lr=0.2 mil=10 nl=20} dout=%Output% loader=Text{col=Label:R4:11 col=Features:R4:0-10 sep=; header+} data=%Data% out=%Output% seed=1	/em:RootMeanSquaredError;/iter:50;/lr:0.2;/nl:20;/mil:10;/v:+;/nt:1	
 

Original file line number	Diff line number	Diff line change
`@@ -162,7 +162,7 @@ public enum EvaluateMetricType`
`162`	`162`	`[Argument(ArgumentType.AtMostOnce,`
`163`	`163`	`HelpText = "Evaluation metrics.",`
`164`	`164`	`ShortName = "em")]`
`165`		`- public EvaluateMetricType EvaluationMetric = EvaluateMetricType.Logloss;`
	`165`	`+ public EvaluateMetricType EvaluationMetric = EvaluateMetricType.Default;`
`166`	`166`
`167`	`167`	`static Options()`
`168`	`168`	`{`
Original file line number	Diff line number	Diff line change
`@@ -95,7 +95,7 @@ public enum EvaluateMetricType`
`95`	`95`	`[Argument(ArgumentType.AtMostOnce,`
`96`	`96`	`HelpText = "Evaluation metrics.",`
`97`	`97`	`ShortName = "em")]`
`98`		`- public EvaluateMetricType EvaluationMetric = EvaluateMetricType.Error;`
	`98`	`+ public EvaluateMetricType EvaluationMetric = EvaluateMetricType.Default;`
`99`	`99`
`100`	`100`	`static Options()`
`101`	`101`	`{`
Original file line number	Diff line number	Diff line change
`@@ -143,7 +143,7 @@ public enum EvaluateMetricType`
`143`	`143`	`[Argument(ArgumentType.AtMostOnce,`
`144`	`144`	`HelpText = "Evaluation metrics.",`
`145`	`145`	`ShortName = "em")]`
`146`		`- public EvaluateMetricType EvaluationMetric = EvaluateMetricType.NormalizedDiscountedCumulativeGain;`
	`146`	`+ public EvaluateMetricType EvaluationMetric = EvaluateMetricType.Default;`
`147`	`147`
`148`	`148`	`static Options()`
`149`	`149`	`{`
Original file line number	Diff line number	Diff line change
`@@ -133,7 +133,7 @@ public enum EvaluateMetricType`
`133`	`133`	`[Argument(ArgumentType.AtMostOnce,`
`134`	`134`	`HelpText = "Evaluation metrics.",`
`135`	`135`	`ShortName = "em")]`
`136`		`- public EvaluateMetricType EvaluationMetric = EvaluateMetricType.RootMeanSquaredError;`
	`136`	`+ public EvaluateMetricType EvaluationMetric = EvaluateMetricType.Default;`
`137`	`137`
`138`	`138`	`static Options()`
`139`	`139`	`{`