Moving IModelCombiner to Ensemble and related changes

src/Microsoft.ML.Core/Prediction/ITrainer.cs

@@ -2,7 +2,6 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-using System.Collections.Generic;
 using Microsoft.ML.Runtime.Data;
 
 namespace Microsoft.ML.Runtime
@@ -94,12 +93,4 @@ public static IPredictor Train(this ITrainer trainer, RoleMappedData trainData)
         public static TPredictor Train<TPredictor>(this ITrainer<TPredictor> trainer, RoleMappedData trainData) where TPredictor : IPredictor
             => trainer.Train(new TrainContext(trainData));
     }
-
-    /// <summary>
-    /// An interface that combines multiple predictors into a single predictor.
-    /// </summary>
-    public interface IModelCombiner
-    {
-        IPredictor CombineModels(IEnumerable<IPredictor> models);
-    }
 }

src/Microsoft.ML.Core/Properties/AssemblyInfo.cs

@@ -6,6 +6,7 @@
 using Microsoft.ML;
 
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.TestFramework" + PublicKey.TestValue)]
+[assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Predictor.Tests" + PublicKey.TestValue)]
 
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Legacy" + PublicKey.Value)]
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Maml" + PublicKey.Value)]

src/Microsoft.ML.Ensemble/Microsoft.ML.Ensemble.csproj

@@ -10,7 +10,6 @@
     <ProjectReference Include="..\Microsoft.ML.Data\Microsoft.ML.Data.csproj" />
     <ProjectReference Include="..\Microsoft.ML.StandardLearners\Microsoft.ML.StandardLearners.csproj" />
     <ProjectReference Include="..\Microsoft.ML.Transforms\Microsoft.ML.Transforms.csproj" />
-    <ProjectReference Include="..\Microsoft.ML.FastTree\Microsoft.ML.FastTree.csproj" />
   </ItemGroup>
 
 </Project>

src/Microsoft.ML.Ensemble/OutputCombiners/MultiStacking.cs

@@ -8,10 +8,8 @@
 using Microsoft.ML.Runtime.Data;
 using Microsoft.ML.Runtime.Ensemble.OutputCombiners;
 using Microsoft.ML.Runtime.EntryPoints;
-using Microsoft.ML.Trainers.FastTree;
 using Microsoft.ML.Runtime.Internal.Internallearn;
 using Microsoft.ML.Runtime.Internal.Utilities;
-using Microsoft.ML.Runtime.Learners;
 using Microsoft.ML.Runtime.Model;
 
 [assembly: LoadableClass(typeof(MultiStacking), typeof(MultiStacking.Arguments), typeof(SignatureCombiner),
@@ -50,17 +48,6 @@ public sealed class Arguments : ArgumentsBase, ISupportMulticlassOutputCombinerF
             internal override IComponentFactory<ITrainer<TVectorPredictor>> GetPredictorFactory() => BasePredictorType;
 
             public IMultiClassOutputCombiner CreateComponent(IHostEnvironment env) => new MultiStacking(env, this);
-
-            public Arguments()
-            {
-                // REVIEW: Perhaps we can have a better non-parametetric learner.
-                BasePredictorType = ComponentFactoryUtils.CreateFromFunction(
-                    env => new Ova(env, new Ova.Arguments()
-                    {
-                        PredictorType = ComponentFactoryUtils.CreateFromFunction(
-                            e => new FastTreeBinaryClassificationTrainer(e, DefaultColumnNames.Label, DefaultColumnNames.Features))
-                    }));
-            }
         }
 
         public MultiStacking(IHostEnvironment env, Arguments args)

src/Microsoft.ML.Ensemble/OutputCombiners/RegressionStacking.cs

@@ -4,10 +4,8 @@
 using System;
 using Microsoft.ML.Runtime;
 using Microsoft.ML.Runtime.CommandLine;
-using Microsoft.ML.Runtime.Data;
 using Microsoft.ML.Runtime.Ensemble.OutputCombiners;
 using Microsoft.ML.Runtime.EntryPoints;
-using Microsoft.ML.Trainers.FastTree;
 using Microsoft.ML.Runtime.Internal.Internallearn;
 using Microsoft.ML.Runtime.Model;
 
@@ -47,12 +45,6 @@ public sealed class Arguments : ArgumentsBase, ISupportRegressionOutputCombinerF
 
             internal override IComponentFactory<ITrainer<TScalarPredictor>> GetPredictorFactory() => BasePredictorType;
 
-            public Arguments()
-            {
-                BasePredictorType = ComponentFactoryUtils.CreateFromFunction(
-                    env => new FastTreeRegressionTrainer(env, DefaultColumnNames.Label, DefaultColumnNames.Features));
-            }
-
             public IRegressionOutputCombiner CreateComponent(IHostEnvironment env) => new RegressionStacking(env, this);
         }
 

src/Microsoft.ML.Ensemble/OutputCombiners/Stacking.cs

@@ -5,10 +5,8 @@
 using System;
 using Microsoft.ML.Runtime;
 using Microsoft.ML.Runtime.CommandLine;
-using Microsoft.ML.Runtime.Data;
 using Microsoft.ML.Runtime.Ensemble.OutputCombiners;
 using Microsoft.ML.Runtime.EntryPoints;
-using Microsoft.ML.Trainers.FastTree;
 using Microsoft.ML.Runtime.Internal.Internallearn;
 using Microsoft.ML.Runtime.Model;
 
@@ -45,12 +43,6 @@ public sealed class Arguments : ArgumentsBase, ISupportBinaryOutputCombinerFacto
 
             internal override IComponentFactory<ITrainer<TScalarPredictor>> GetPredictorFactory() => BasePredictorType;
 
-            public Arguments()
-            {
-                BasePredictorType = ComponentFactoryUtils.CreateFromFunction(
-                    env => new FastTreeBinaryClassificationTrainer(env, DefaultColumnNames.Label, DefaultColumnNames.Features));
-            }
-
             public IBinaryOutputCombiner CreateComponent(IHostEnvironment env) => new Stacking(env, this);
         }
 

src/Microsoft.ML.Ensemble/Trainer/IModelCombiner.cs

@@ -0,0 +1,17 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Collections.Generic;
+using Microsoft.ML.Runtime;
+
+namespace Microsoft.ML.Runtime.Ensemble
+{
+    /// <summary>
+    /// An interface that combines multiple predictors into a single predictor.
+    /// </summary>
+    public interface IModelCombiner
+    {
+        IPredictor CombineModels(IEnumerable<IPredictor> models);
+    }
+}

src/Microsoft.ML.FastTree/FastTree.cs

@@ -55,7 +55,7 @@ public abstract class FastTreeTrainerBase<TArgs, TTransformer, TModel> :
     {
         protected readonly TArgs Args;
         protected readonly bool AllowGC;
-        protected Ensemble TrainedEnsemble;
+        protected TreeEnsemble TrainedEnsemble;
         protected int FeatureCount;
         protected RoleMappedData ValidData;
         protected IParallelTraining ParallelTraining;
@@ -76,7 +76,7 @@ public abstract class FastTreeTrainerBase<TArgs, TTransformer, TModel> :
         protected double[] InitValidScores;
         protected double[][] InitTestScores;
         //protected int Iteration;
-        protected Ensemble Ensemble;
+        protected TreeEnsemble Ensemble;
 
         protected bool HasValidSet => ValidSet != null;
 
@@ -478,7 +478,7 @@ protected bool AreSamplesWeighted(IChannel ch)
 
         private void InitializeEnsemble()
         {
-            Ensemble = new Ensemble();
+            Ensemble = new TreeEnsemble();
         }
 
         /// <summary>
@@ -914,7 +914,7 @@ internal abstract class DataConverter
         /// of features we actually trained on. This can be null in the event that no filtering
         /// occurred.
         /// </summary>
-        /// <seealso cref="Ensemble.RemapFeatures"/>
+        /// <seealso cref="TreeEnsemble.RemapFeatures"/>
         public int[] FeatureMap;
 
         protected readonly IHost Host;
@@ -2810,7 +2810,7 @@ public abstract class FastTreePredictionWrapper :
         ISingleCanSaveOnnx
     {
         //The below two properties are necessary for tree Visualizer
-        public Ensemble TrainedEnsemble { get; }
+        public TreeEnsemble TrainedEnsemble { get; }
         public int NumTrees => TrainedEnsemble.NumTrees;
 
         // Inner args is used only for documentation purposes when saving comments to INI files.
@@ -2834,7 +2834,7 @@ public abstract class FastTreePredictionWrapper :
         public bool CanSavePfa => true;
         public bool CanSaveOnnx(OnnxContext ctx) => true;
 
-        protected FastTreePredictionWrapper(IHostEnvironment env, string name, Ensemble trainedEnsemble, int numFeatures, string innerArgs)
+        protected FastTreePredictionWrapper(IHostEnvironment env, string name, TreeEnsemble trainedEnsemble, int numFeatures, string innerArgs)
             : base(env, name)
         {
             Host.CheckValue(trainedEnsemble, nameof(trainedEnsemble));
@@ -2871,7 +2871,7 @@ protected FastTreePredictionWrapper(IHostEnvironment env, string name, ModelLoad
             if (ctx.Header.ModelVerWritten >= VerCategoricalSplitSerialized)
                 categoricalSplits = true;
 
-            TrainedEnsemble = new Ensemble(ctx, usingDefaultValues, categoricalSplits);
+            TrainedEnsemble = new TreeEnsemble(ctx, usingDefaultValues, categoricalSplits);
             MaxSplitFeatIdx = FindMaxFeatureIndex(TrainedEnsemble);
 
             InnerArgs = ctx.LoadStringOrNull();
@@ -3264,7 +3264,7 @@ public void GetFeatureWeights(ref VBuffer<Float> weights)
             bldr.GetResult(ref weights);
         }
 
-        private static int FindMaxFeatureIndex(Ensemble ensemble)
+        private static int FindMaxFeatureIndex(TreeEnsemble ensemble)
         {
             int ifeatMax = 0;
             for (int i = 0; i < ensemble.NumTrees; i++)

src/Microsoft.ML.FastTree/FastTreeClassification.cs

@@ -68,7 +68,7 @@ private static VersionInfo GetVersionInfo()
 
         protected override uint VerCategoricalSplitSerialized => 0x00010005;
 
-        internal FastTreeBinaryPredictor(IHostEnvironment env, Ensemble trainedEnsemble, int featureCount, string innerArgs)
+        internal FastTreeBinaryPredictor(IHostEnvironment env, TreeEnsemble trainedEnsemble, int featureCount, string innerArgs)
             : base(env, RegistrationName, trainedEnsemble, featureCount, innerArgs)
         {
         }

src/Microsoft.ML.FastTree/FastTreeRanking.cs

@@ -1114,7 +1114,7 @@ private static VersionInfo GetVersionInfo()
 
         protected override uint VerCategoricalSplitSerialized => 0x00010005;
 
-        internal FastTreeRankingPredictor(IHostEnvironment env, Ensemble trainedEnsemble, int featureCount, string innerArgs)
+        internal FastTreeRankingPredictor(IHostEnvironment env, TreeEnsemble trainedEnsemble, int featureCount, string innerArgs)
             : base(env, RegistrationName, trainedEnsemble, featureCount, innerArgs)
         {
         }

src/Microsoft.ML.FastTree/FastTreeRegression.cs

@@ -462,7 +462,7 @@ private static VersionInfo GetVersionInfo()
 
         protected override uint VerCategoricalSplitSerialized => 0x00010005;
 
-        internal FastTreeRegressionPredictor(IHostEnvironment env, Ensemble trainedEnsemble, int featureCount, string innerArgs)
+        internal FastTreeRegressionPredictor(IHostEnvironment env, TreeEnsemble trainedEnsemble, int featureCount, string innerArgs)
             : base(env, RegistrationName, trainedEnsemble, featureCount, innerArgs)
         {
         }

src/Microsoft.ML.FastTree/FastTreeTweedie.cs

@@ -465,7 +465,7 @@ private static VersionInfo GetVersionInfo()
 
         protected override uint VerCategoricalSplitSerialized => 0x00010003;
 
-        internal FastTreeTweediePredictor(IHostEnvironment env, Ensemble trainedEnsemble, int featureCount, string innerArgs)
+        internal FastTreeTweediePredictor(IHostEnvironment env, TreeEnsemble trainedEnsemble, int featureCount, string innerArgs)
             : base(env, RegistrationName, trainedEnsemble, featureCount, innerArgs)
         {
         }

src/Microsoft.ML.FastTree/Microsoft.ML.FastTree.csproj

@@ -11,6 +11,7 @@
     <ProjectReference Include="..\Microsoft.ML.Core\Microsoft.ML.Core.csproj" />
     <ProjectReference Include="..\Microsoft.ML.CpuMath\Microsoft.ML.CpuMath.csproj" />
     <ProjectReference Include="..\Microsoft.ML.Data\Microsoft.ML.Data.csproj" />
+    <ProjectReference Include="..\Microsoft.ML.Ensemble\Microsoft.ML.Ensemble.csproj" />
   </ItemGroup>
 
 </Project>

src/Microsoft.ML.FastTree/RandomForestClassification.cs

@@ -78,7 +78,7 @@ private static VersionInfo GetVersionInfo()
         /// </summary>
         public override PredictionKind PredictionKind => PredictionKind.BinaryClassification;
 
-        public FastForestClassificationPredictor(IHostEnvironment env, Ensemble trainedEnsemble, int featureCount, string innerArgs)
+        public FastForestClassificationPredictor(IHostEnvironment env, TreeEnsemble trainedEnsemble, int featureCount, string innerArgs)
             : base(env, RegistrationName, trainedEnsemble, featureCount, innerArgs)
         { }
 

src/Microsoft.ML.FastTree/RandomForestRegression.cs

@@ -59,7 +59,7 @@ private static VersionInfo GetVersionInfo()
 
         protected override uint VerCategoricalSplitSerialized => 0x00010006;
 
-        public FastForestRegressionPredictor(IHostEnvironment env, Ensemble trainedEnsemble, int featureCount, string innerArgs, int samplesCount)
+        public FastForestRegressionPredictor(IHostEnvironment env, TreeEnsemble trainedEnsemble, int featureCount, string innerArgs, int samplesCount)
             : base(env, RegistrationName, trainedEnsemble, featureCount, innerArgs)
         {
             _quantileSampleCount = samplesCount;

src/Microsoft.ML.FastTree/Training/BaggingProvider.cs

@@ -75,7 +75,7 @@ public int GetBagCount(int numTrees, int bagSize)
         // Divides output values of leaves to bag count.
         // This brings back the final scores generated by model on a same
         // range as when we didn't use bagging
-        public void ScaleEnsembleLeaves(int numTrees, int bagSize, Ensemble ensemble)
+        public void ScaleEnsembleLeaves(int numTrees, int bagSize, TreeEnsemble ensemble)
         {
             int bagCount = GetBagCount(numTrees, bagSize);
             for (int t = 0; t < ensemble.NumTrees; t++)

src/Microsoft.ML.FastTree/Training/EnsembleCompression/IEnsembleCompressor.cs

@@ -14,8 +14,8 @@ public interface IEnsembleCompressor<TLabel>
 
         void SetTreeScores(int idx, double[] scores);
 
-        bool Compress(IChannel ch, Ensemble ensemble, double[] trainScores, int bestIteration, int maxTreesAfterCompression);
+        bool Compress(IChannel ch, TreeEnsemble ensemble, double[] trainScores, int bestIteration, int maxTreesAfterCompression);
 
-        Ensemble GetCompressedEnsemble();
+        TreeEnsemble GetCompressedEnsemble();
     }
 }

src/Microsoft.ML.FastTree/Training/EnsembleCompression/LassoBasedEnsembleCompressor.cs

@@ -51,7 +51,7 @@ public class LassoBasedEnsembleCompressor : IEnsembleCompressor<short>
 
         private Dataset _trainSet;
         private short[] _labels;
-        private Ensemble _compressedEnsemble;
+        private TreeEnsemble _compressedEnsemble;
         private int[] _sampleObservationIndices;
         private Random _rnd;
 
@@ -458,9 +458,9 @@ private LassoFit GetLassoFit(IChannel ch, int maxAllowedFeaturesPerModel)
             return fit;
         }
 
-        private Ensemble GetEnsembleFromSolution(LassoFit fit, int solutionIdx, Ensemble originalEnsemble)
+        private TreeEnsemble GetEnsembleFromSolution(LassoFit fit, int solutionIdx, TreeEnsemble originalEnsemble)
         {
-            Ensemble ensemble = new Ensemble();
+            TreeEnsemble ensemble = new TreeEnsemble();
 
             int weightsCount = fit.NumberOfWeights[solutionIdx];
             for (int i = 0; i < weightsCount; i++)
@@ -534,7 +534,7 @@ private unsafe void LoadTargets(double[] trainScores, int bestIteration)
             }
         }
 
-        public bool Compress(IChannel ch, Ensemble ensemble, double[] trainScores, int bestIteration, int maxTreesAfterCompression)
+        public bool Compress(IChannel ch, TreeEnsemble ensemble, double[] trainScores, int bestIteration, int maxTreesAfterCompression)
         {
             LoadTargets(trainScores, bestIteration);
 
@@ -552,7 +552,7 @@ public bool Compress(IChannel ch, Ensemble ensemble, double[] trainScores, int b
             return true;
         }
 
-        public Ensemble GetCompressedEnsemble()
+        public TreeEnsemble GetCompressedEnsemble()
         {
             return _compressedEnsemble;
         }

src/Microsoft.ML.FastTree/Training/OptimizationAlgorithms/AcceleratedGradientDescent.cs

@@ -9,7 +9,7 @@ namespace Microsoft.ML.Trainers.FastTree.Internal
     //Accelerated gradient descent score tracker
     public class AcceleratedGradientDescent : GradientDescent
     {
-        public AcceleratedGradientDescent(Ensemble ensemble, Dataset trainData, double[] initTrainScores, IGradientAdjuster gradientWrapper)
+        public AcceleratedGradientDescent(TreeEnsemble ensemble, Dataset trainData, double[] initTrainScores, IGradientAdjuster gradientWrapper)
             : base(ensemble, trainData, initTrainScores, gradientWrapper)
         {
             UseFastTrainingScoresUpdate = false;

src/Microsoft.ML.FastTree/Training/OptimizationAlgorithms/ConjugateGradientDescent.cs

@@ -13,7 +13,7 @@ public class ConjugateGradientDescent : GradientDescent
         private double[] _currentGradient;
         private double[] _currentDk;
 
-        public ConjugateGradientDescent(Ensemble ensemble, Dataset trainData, double[] initTrainScores, IGradientAdjuster gradientWrapper)
+        public ConjugateGradientDescent(TreeEnsemble ensemble, Dataset trainData, double[] initTrainScores, IGradientAdjuster gradientWrapper)
             : base(ensemble, trainData, initTrainScores, gradientWrapper)
         {
             _currentDk = new double[trainData.NumDocs];

src/Microsoft.ML.FastTree/Training/OptimizationAlgorithms/GradientDescent.cs

@@ -22,7 +22,7 @@ public class GradientDescent : OptimizationAlgorithm
         private double[] _droppedScores;
         private double[] _scores;
 
-        public GradientDescent(Ensemble ensemble, Dataset trainData, double[] initTrainScores, IGradientAdjuster gradientWrapper)
+        public GradientDescent(TreeEnsemble ensemble, Dataset trainData, double[] initTrainScores, IGradientAdjuster gradientWrapper)
             : base(ensemble, trainData, initTrainScores)
         {
             _gradientWrapper = gradientWrapper;

src/Microsoft.ML.FastTree/Training/OptimizationAlgorithms/NoOptimizationAlgorithm.cs

@@ -14,7 +14,7 @@ public class RandomForestOptimizer : GradientDescent
     {
         private IGradientAdjuster _gradientWrapper;
         // REVIEW: When the FastTree appliation is decoupled with tree learner and boosting logic, this class should be removed.
-        public RandomForestOptimizer(Ensemble ensemble, Dataset trainData, double[] initTrainScores, IGradientAdjuster gradientWrapper)
+        public RandomForestOptimizer(TreeEnsemble ensemble, Dataset trainData, double[] initTrainScores, IGradientAdjuster gradientWrapper)
             : base(ensemble, trainData, initTrainScores, gradientWrapper)
         {
             _gradientWrapper = gradientWrapper;

src/Microsoft.ML.FastTree/Training/OptimizationAlgorithms/OptimizationAlgorithm.cs

@@ -26,7 +26,7 @@ public abstract class OptimizationAlgorithm
         public delegate void PreScoreUpdateHandler(IChannel ch);
         public PreScoreUpdateHandler PreScoreUpdateEvent;
 
-        public Ensemble Ensemble;
+        public TreeEnsemble Ensemble;
 
         public ScoreTracker TrainingScores;
         public List<ScoreTracker> TrackedScores;
@@ -37,7 +37,7 @@ public abstract class OptimizationAlgorithm
         public Random DropoutRng;
         public bool UseFastTrainingScoresUpdate;
 
-        public OptimizationAlgorithm(Ensemble ensemble, Dataset trainData, double[] initTrainScores)
+        public OptimizationAlgorithm(TreeEnsemble ensemble, Dataset trainData, double[] initTrainScores)
         {
             Ensemble = ensemble;
             TrainingScores = ConstructScoreTracker("train", trainData, initTrainScores);