Added Microsoft.ML.Benchmarks Project

Microsoft.ML.sln

@@ -101,6 +101,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Microsoft.ML.Parquet", "Mic
 		pkg\Microsoft.ML.Parquet\Microsoft.ML.Parquet.nupkgproj = pkg\Microsoft.ML.Parquet\Microsoft.ML.Parquet.nupkgproj
 	EndProjectSection
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Benchmarks", "test\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj", "{7A9DB75F-2CA5-4184-9EF5-1F17EB39483F}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -195,6 +197,10 @@ Global
 		{55C8122D-79EA-48AB-85D0-EB551FC1C427}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{55C8122D-79EA-48AB-85D0-EB551FC1C427}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{55C8122D-79EA-48AB-85D0-EB551FC1C427}.Release|Any CPU.Build.0 = Release|Any CPU
+		{7A9DB75F-2CA5-4184-9EF5-1F17EB39483F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{7A9DB75F-2CA5-4184-9EF5-1F17EB39483F}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{7A9DB75F-2CA5-4184-9EF5-1F17EB39483F}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{7A9DB75F-2CA5-4184-9EF5-1F17EB39483F}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -228,6 +234,7 @@ Global
 		{2DEFC784-F2B5-44EA-ABBB-0DCF3E689DAC} = {E20AF96D-3F66-4065-8A89-BEE479D74536}
 		{DEC8F776-49F7-4D87-836C-FE4DC057D08C} = {D3D38B03-B557-484D-8348-8BADEE4DF592}
 		{6C95FC87-F5F2-4EEF-BB97-567F2F5DD141} = {D3D38B03-B557-484D-8348-8BADEE4DF592}
+		{7A9DB75F-2CA5-4184-9EF5-1F17EB39483F} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {41165AF1-35BB-4832-A189-73060F82B01D}

test/Microsoft.ML.Benchmarks/Microsoft.ML.Benchmarks.csproj

@@ -0,0 +1,23 @@
+<Project Sdk="Microsoft.NET.Sdk" ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <LangVersion>7.2</LangVersion>    
+    <StartupObject>Microsoft.ML.Benchmarks.Program</StartupObject>
+    <TargetFramework>netcoreapp2.0</TargetFramework>
+  </PropertyGroup>
+  <ItemGroup>
+    <Compile Remove="BenchmarkDotNet.Artifacts\**" />
+    <EmbeddedResource Remove="BenchmarkDotNet.Artifacts\**" />
+    <None Remove="BenchmarkDotNet.Artifacts\**" />
+  </ItemGroup>
+  <ItemGroup>
+    <PackageReference Include="BenchmarkDotNet" Version="0.10.14" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\..\src\Microsoft.ML.StandardLearners\Microsoft.ML.StandardLearners.csproj" />
+    <ProjectReference Include="..\..\src\Microsoft.ML\Microsoft.ML.csproj" />
+  </ItemGroup>
+  <ItemGroup>
+    <NativeAssemblyReference Include="CpuMathNative" />
+  </ItemGroup>
+</Project>

test/Microsoft.ML.Benchmarks/Program.cs

@@ -0,0 +1,89 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Diagnosers;
+using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Running;
+using BenchmarkDotNet.Columns;
+using BenchmarkDotNet.Reports;
+using BenchmarkDotNet.Toolchains.CsProj;
+using BenchmarkDotNet.Toolchains.InProcess;
+using System;
+using System.IO;
+using Microsoft.ML.Models;
+using Microsoft.ML.Runtime.Api;
+using Microsoft.ML.Trainers;
+using Microsoft.ML.Transforms;
+using Microsoft.ML.Benchmarks;
+
+namespace Microsoft.ML.Benchmarks
+{
+    class Program
+    {
+        /// <summary>
+        /// execute dotnet run -c Release and choose the benchmarks you want to run
+        /// </summary>
+        /// <param name="args"></param>
+        static void Main(string[] args)
+        {
+            BenchmarkSwitcher
+                .FromAssembly(typeof(Program).Assembly)
+                .Run(null, CreateClrVsCoreConfig());
+        }
+
+        private static IConfig CreateClrVsCoreConfig()
+        {
+            var config = DefaultConfig.Instance.With(
+                Job.ShortRun.
+                With(InProcessToolchain.Instance)).
+                With(new ClassificationMetricsColumn("AccuracyMacro", "Macro-average accuracy of the model")).
+                With(MemoryDiagnoser.Default);
+            return config;
+        }
+
+        internal static string GetDataPath(string name)
+            => Path.GetFullPath(Path.Combine(_dataRoot, name));
+
+        static readonly string _dataRoot;
+        static Program()
+        {
+            var currentAssemblyLocation = new FileInfo(typeof(Program).Assembly.Location);
+            var rootDir = currentAssemblyLocation.Directory.Parent.Parent.Parent.Parent.FullName;
+            _dataRoot = Path.Combine(rootDir, "test", "data");
+        }
+    }
+
+    public class ClassificationMetricsColumn : IColumn
+    {
+        string _metricName;
+        string _legend;
+
+        public ClassificationMetricsColumn(string metricName, string legend)
+        {
+            _metricName = metricName;
+            _legend = legend;
+        }
+
+        public string ColumnName => _metricName;
+        public string Id => _metricName;
+        public string Legend => _legend;
+        public bool IsNumeric => true;
+        public bool IsDefault(Summary summary, Benchmark benchmark) => true;
+        public bool IsAvailable(Summary summary) => true;
+        public bool AlwaysShow => true;
+        public ColumnCategory Category => ColumnCategory.Custom;
+        public int PriorityInCategory => 1;
+        public UnitType UnitType => UnitType.Dimensionless;
+
+        public string GetValue(Summary summary, Benchmark benchmark, ISummaryStyle style)
+        {
+            var property = typeof(ClassificationMetrics).GetProperty(_metricName);
+            return property.GetValue(StochasticDualCoordinateAscentClassifierBench.s_metrics).ToString();
+        }
+        public string GetValue(Summary summary, Benchmark benchmark) => GetValue(summary, benchmark, null);
+
+        public override string ToString() => ColumnName;
+    }
+}

test/Microsoft.ML.Benchmarks/StochasticDualCoordinateAscentClassifierBench.cs

@@ -0,0 +1,107 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Running;
+using Microsoft.ML.Models;
+using Microsoft.ML.Runtime.Api;
+using Microsoft.ML.Trainers;
+using Microsoft.ML.Transforms;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace Microsoft.ML.Benchmarks
+{
+    public class StochasticDualCoordinateAscentClassifierBench
+    {
+        internal static ClassificationMetrics s_metrics;
+        private static PredictionModel<IrisData, IrisPrediction> s_trainedModel;
+        private static string s_dataPath;
+        private static IrisData[][] s_batches;
+        private static readonly int[] s_batchSizes = new int[] { 1, 2, 5 };
+        private readonly Random r = new Random(0);
+        private readonly static IrisData s_example = new IrisData()
+        {
+            SepalLength = 3.3f,
+            SepalWidth = 1.6f,
+            PetalLength = 0.2f,
+            PetalWidth = 5.1f,
+        };
+
+        [Benchmark]
+        public PredictionModel<IrisData, IrisPrediction> TrainIris() => TrainCore();
+
+        [Benchmark]
+        public float[] PredictIris() => s_trainedModel.Predict(s_example).PredictedLabels;
+
+        [Benchmark]
+        public IEnumerable<IrisPrediction> PredictIrisBatchOf1() => s_trainedModel.Predict(s_batches[0]);
+        [Benchmark]
+        public IEnumerable<IrisPrediction> PredictIrisBatchOf2() => s_trainedModel.Predict(s_batches[1]);
+        [Benchmark]
+        public IEnumerable<IrisPrediction> PredictIrisBatchOf5() => s_trainedModel.Predict(s_batches[2]);
+
+        [GlobalSetup]
+        public void Setup()
+        {
+            s_dataPath = Program.GetDataPath("iris.txt");
+            s_trainedModel = TrainCore();
+            IrisPrediction prediction = s_trainedModel.Predict(s_example);
+
+            var testData = new TextLoader<IrisData>(s_dataPath, useHeader: true, separator: "tab");
+            var evaluator = new ClassificationEvaluator();
+            s_metrics = evaluator.Evaluate(s_trainedModel, testData);
+
+            s_batches = new IrisData[s_batchSizes.Length][];
+            for (int i = 0; i < s_batches.Length; i++)
+            {
+                var batch = new IrisData[s_batchSizes[i]];
+                s_batches[i] = batch;
+                for (int bi = 0; bi < batch.Length; bi++)
+                {
+                    batch[bi] = s_example;
+                }
+            }
+        }
+
+        private static PredictionModel<IrisData, IrisPrediction> TrainCore()
+        {
+            var pipeline = new LearningPipeline();
+
+            pipeline.Add(new TextLoader<IrisData>(s_dataPath, useHeader: true, separator: "tab"));
+            pipeline.Add(new ColumnConcatenator(outputColumn: "Features",
+                "SepalLength", "SepalWidth", "PetalLength", "PetalWidth"));
+
+            pipeline.Add(new StochasticDualCoordinateAscentClassifier());
+
+            PredictionModel<IrisData, IrisPrediction> model = pipeline.Train<IrisData, IrisPrediction>();
+            return model;
+        }
+
+        public class IrisData
+        {
+            [Column("0")]
+            public float Label;
+
+            [Column("1")]
+            public float SepalLength;
+
+            [Column("2")]
+            public float SepalWidth;
+
+            [Column("3")]
+            public float PetalLength;
+
+            [Column("4")]
+            public float PetalWidth;
+        }
+
+        public class IrisPrediction
+        {
+            [ColumnName("Score")]
+            public float[] PredictedLabels;
+        }
+    }
+}