diff --git a/.gitignore b/.gitignore index 6b769ea510..cb62c74d37 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ # Tool Runtime Dir /[Tt]ools/ /.dotnet/ +/.packages/ # User-specific files *.suo diff --git a/.vsts-dotnet-ci.yml b/.vsts-dotnet-ci.yml index 5d5bfe7c0a..d3315c00c5 100644 --- a/.vsts-dotnet-ci.yml +++ b/.vsts-dotnet-ci.yml @@ -11,42 +11,42 @@ resources: image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-mlnet-20200515184230-2c829e8 jobs: -# - template: /build/ci/job-template.yml -# parameters: -# name: Centos_x64_NetCoreApp31 -# buildScript: ./build.sh -# container: CentosContainer -# customMatrixes: -# Debug_Build: -# _configuration: Debug-netcoreapp3_1 -# _config_short: DI -# _includeBenchmarkData: false -# _targetFramework: netcoreapp3.1 -# Release_Build: -# _configuration: Release-netcoreapp3_1 -# _config_short: RI -# _includeBenchmarkData: true -# _targetFramework: netcoreapp3.1 -# innerLoop: true -# pool: -# name: Hosted Ubuntu 1604 +- template: /build/ci/job-template.yml + parameters: + name: Centos_x64_NetCoreApp31 + buildScript: ./build.sh + container: CentosContainer + customMatrixes: + Debug_Build: + _configuration: Debug-netcoreapp3_1 + _config_short: DI + _includeBenchmarkData: false + _targetFramework: netcoreapp3.1 + Release_Build: + _configuration: Release-netcoreapp3_1 + _config_short: RI + _includeBenchmarkData: true + _targetFramework: netcoreapp3.1 + innerLoop: true + pool: + name: Hosted Ubuntu 1604 -# - template: /build/ci/job-template.yml -# parameters: -# name: Ubuntu_x64_NetCoreApp21 -# buildScript: ./build.sh -# container: UbuntuContainer -# innerLoop: true -# pool: -# name: Hosted Ubuntu 1604 +- template: /build/ci/job-template.yml + parameters: + name: Ubuntu_x64_NetCoreApp21 + buildScript: ./build.sh + container: UbuntuContainer + innerLoop: true + pool: + name: Hosted Ubuntu 1604 -# - template: /build/ci/job-template.yml -# parameters: -# name: MacOS_x64_NetCoreApp21 -# buildScript: ./build.sh -# innerLoop: true -# pool: -# name: Hosted macOS +- template: /build/ci/job-template.yml + parameters: + name: MacOS_x64_NetCoreApp21 + buildScript: ./build.sh + innerLoop: true + pool: + name: Hosted macOS - template: /build/ci/job-template.yml parameters: diff --git a/Microsoft.ML.sln b/Microsoft.ML.sln index 4978dd67c7..fe00b1577a 100644 --- a/Microsoft.ML.sln +++ b/Microsoft.ML.sln @@ -48,7 +48,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "pkg", "pkg", "{D3D38B03-B55 pkg\Directory.Build.props = pkg\Directory.Build.props EndProjectSection EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Benchmarks", "test\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj", "{7A9DB75F-2CA5-4184-9EF5-1F17EB39483F}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.PerformanceTests", "test\Microsoft.ML.PerformanceTests\Microsoft.ML.PerformanceTests.csproj", "{7A9DB75F-2CA5-4184-9EF5-1F17EB39483F}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Maml", "src\Microsoft.ML.Maml\Microsoft.ML.Maml.csproj", "{64F40A0D-D4C2-4AA7-8470-E9CC437827E4}" EndProject diff --git a/build.sh b/build.sh index fa63e891e3..87cb508fc7 100755 --- a/build.sh +++ b/build.sh @@ -10,4 +10,4 @@ while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symli done DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" -"$DIR/eng/common/build.sh" --restore --build --pack --warnAsError false "$@" +"$DIR/eng/common/build.sh" --restore --build --warnAsError false "$@" diff --git a/build/ci/job-template.yml b/build/ci/job-template.yml index 1358f25296..a7039757c4 100644 --- a/build/ci/job-template.yml +++ b/build/ci/job-template.yml @@ -58,9 +58,10 @@ jobs: brew untap local/openssl | brew untap local/python2 displayName: MacOS Homebrew bug Workaround + # Extra MacOS step required to install OS-specific dependencies - ${{ if eq(parameters.pool.name, 'Hosted macOS') }}: - script: brew update && brew unlink python@3.8 && brew install mono-libgdiplus && brew install $(Build.SourcesDirectory)/build/libomp.rb && brew link libomp --force - displayName: Install build dependencies + displayName: Install MacOS build dependencies - ${{ if and( eq(parameters.nightlyBuild, 'true'), eq(parameters.pool.name, 'Hosted Ubuntu 1604')) }}: - bash: echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$(nightlyBuildRunPath):$LD_LIBRARY_PATH" displayName: Set LD_LIBRARY_PATH for Ubuntu and CentOS to locate Native shared library in current running path @@ -186,5 +187,5 @@ jobs: artifactName: ${{ parameters.name }} $(_config_short) artifactType: container - ${{ if eq(parameters.nightlyBuild, 'false') }}: - - script: ${{ parameters.buildScript }} -pack -ci + - script: ${{ parameters.buildScript }} -pack -ci displayName: Build Packages diff --git a/src/Microsoft.ML.Core/Properties/AssemblyInfo.cs b/src/Microsoft.ML.Core/Properties/AssemblyInfo.cs index e48aebf0e0..a41256c515 100644 --- a/src/Microsoft.ML.Core/Properties/AssemblyInfo.cs +++ b/src/Microsoft.ML.Core/Properties/AssemblyInfo.cs @@ -13,7 +13,7 @@ [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.InferenceTesting" + PublicKey.Value)] [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.OnnxTransformerTest" + PublicKey.Value)] [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.AutoML.Tests" + PublicKey.Value)] -[assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Benchmarks" + PublicKey.Value)] +[assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.PerformanceTests" + PublicKey.Value)] [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.EntryPoints" + PublicKey.Value)] [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Maml" + PublicKey.Value)] diff --git a/src/Microsoft.ML.CpuMath/AssemblyInfo.cs b/src/Microsoft.ML.CpuMath/AssemblyInfo.cs index d5dd10bbe9..8145826847 100644 --- a/src/Microsoft.ML.CpuMath/AssemblyInfo.cs +++ b/src/Microsoft.ML.CpuMath/AssemblyInfo.cs @@ -15,6 +15,6 @@ [assembly: InternalsVisibleTo("Microsoft.ML.Sweeper" + PublicKey.Value)] [assembly: InternalsVisibleTo("Microsoft.ML.TimeSeries" + PublicKey.Value)] [assembly: InternalsVisibleTo("Microsoft.ML.Transforms" + PublicKey.Value)] -[assembly: InternalsVisibleTo("Microsoft.ML.Benchmarks.Tests" + PublicKey.Value)] +[assembly: InternalsVisibleTo("Microsoft.ML.PerformanceTests.Tests" + PublicKey.Value)] [assembly: WantsToBeBestFriends] diff --git a/src/Microsoft.ML.Data/Properties/AssemblyInfo.cs b/src/Microsoft.ML.Data/Properties/AssemblyInfo.cs index 91e9a20caf..d8ede92abc 100644 --- a/src/Microsoft.ML.Data/Properties/AssemblyInfo.cs +++ b/src/Microsoft.ML.Data/Properties/AssemblyInfo.cs @@ -12,10 +12,10 @@ [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.OnnxTransformerTest" + PublicKey.Value)] [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Predictor.Tests" + PublicKey.Value)] [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.TimeSeries.Tests" + PublicKey.Value)] -[assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Benchmarks" + PublicKey.Value)] +[assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.PerformanceTests" + PublicKey.Value)] [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.AutoML.Tests" + PublicKey.Value)] -[assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Benchmarks" + PublicKey.Value)] +[assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.PerformanceTests" + PublicKey.Value)] [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.EntryPoints" + PublicKey.Value)] [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Maml" + PublicKey.Value)] [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.ResultProcessor" + PublicKey.Value)] diff --git a/src/Microsoft.ML.Maml/Properties/AssemblyInfo.cs b/src/Microsoft.ML.Maml/Properties/AssemblyInfo.cs index 5a98073dea..bb1e6f1fdc 100644 --- a/src/Microsoft.ML.Maml/Properties/AssemblyInfo.cs +++ b/src/Microsoft.ML.Maml/Properties/AssemblyInfo.cs @@ -6,7 +6,7 @@ using Microsoft.ML; [assembly: InternalsVisibleTo("Microsoft.ML.TestFramework" + PublicKey.Value)] -[assembly: InternalsVisibleTo("Microsoft.ML.Benchmarks" + PublicKey.Value)] +[assembly: InternalsVisibleTo("Microsoft.ML.PerformanceTests" + PublicKey.Value)] [assembly: InternalsVisibleTo("Microsoft.ML.ResultProcessor" + PublicKey.Value)] [assembly: InternalsVisibleTo("RunTests" + InternalPublicKey.Value)] diff --git a/src/Redist/Microsoft.ML.DnnImageFeaturizer.ModelRedist/Microsoft.ML.DnnImageFeaturizer.ModelRedist.proj b/src/Redist/Microsoft.ML.DnnImageFeaturizer.ModelRedist/Microsoft.ML.DnnImageFeaturizer.ModelRedist.proj index 73e591b2f8..89d6108ac5 100644 --- a/src/Redist/Microsoft.ML.DnnImageFeaturizer.ModelRedist/Microsoft.ML.DnnImageFeaturizer.ModelRedist.proj +++ b/src/Redist/Microsoft.ML.DnnImageFeaturizer.ModelRedist/Microsoft.ML.DnnImageFeaturizer.ModelRedist.proj @@ -123,4 +123,6 @@ + + \ No newline at end of file diff --git a/test/Microsoft.ML.Benchmarks.Tests/BenchmarksTest.cs b/test/Microsoft.ML.Benchmarks.Tests/BenchmarksTest.cs index 88f975927a..285aad7845 100644 --- a/test/Microsoft.ML.Benchmarks.Tests/BenchmarksTest.cs +++ b/test/Microsoft.ML.Benchmarks.Tests/BenchmarksTest.cs @@ -10,13 +10,13 @@ using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Running; -using Microsoft.ML.Benchmarks.Harness; +using Microsoft.ML.PerformanceTests.Harness; using Microsoft.ML.TestFramework; using Microsoft.ML.TestFramework.Attributes; using Xunit; using Xunit.Abstractions; -namespace Microsoft.ML.Benchmarks.Tests +namespace Microsoft.ML.PerformanceTests.Tests { public class TestConfig : RecommendedConfig { diff --git a/test/Microsoft.ML.Benchmarks.Tests/Microsoft.ML.Benchmarks.Tests.csproj b/test/Microsoft.ML.Benchmarks.Tests/Microsoft.ML.Benchmarks.Tests.csproj index ad555d7b49..9df7fd4b36 100644 --- a/test/Microsoft.ML.Benchmarks.Tests/Microsoft.ML.Benchmarks.Tests.csproj +++ b/test/Microsoft.ML.Benchmarks.Tests/Microsoft.ML.Benchmarks.Tests.csproj @@ -1,7 +1,7 @@  - + diff --git a/test/Microsoft.ML.Benchmarks/Program.cs b/test/Microsoft.ML.Benchmarks/Program.cs deleted file mode 100644 index 13d41b1341..0000000000 --- a/test/Microsoft.ML.Benchmarks/Program.cs +++ /dev/null @@ -1,27 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System.Globalization; -using System.Threading; -using BenchmarkDotNet.Running; - -namespace Microsoft.ML.Benchmarks -{ - class Program - { - /// - /// execute dotnet run -c Release and choose the benchmarks you want to run - /// - /// - static void Main(string[] args) - { - // enforce Neutral Language as "en-us" because the input data files use dot as decimal separator (and it fails for cultures with ",") - Thread.CurrentThread.CurrentCulture = CultureInfo.InvariantCulture; - - BenchmarkSwitcher - .FromAssembly(typeof(Program).Assembly) - .Run(args, new RecommendedConfig()); - } - } -} diff --git a/test/Microsoft.ML.CpuMath.PerformanceTests/Microsoft.ML.CpuMath.PerformanceTests.csproj b/test/Microsoft.ML.CpuMath.PerformanceTests/Microsoft.ML.CpuMath.PerformanceTests.csproj index 72982dc251..4e79957fd5 100644 --- a/test/Microsoft.ML.CpuMath.PerformanceTests/Microsoft.ML.CpuMath.PerformanceTests.csproj +++ b/test/Microsoft.ML.CpuMath.PerformanceTests/Microsoft.ML.CpuMath.PerformanceTests.csproj @@ -10,6 +10,8 @@ --> true + false + true diff --git a/test/Microsoft.ML.Benchmarks/BenchmarkBase.cs b/test/Microsoft.ML.PerformanceTests/BenchmarkBase.cs similarity index 99% rename from test/Microsoft.ML.Benchmarks/BenchmarkBase.cs rename to test/Microsoft.ML.PerformanceTests/BenchmarkBase.cs index 4632ca5ed6..a648ca303a 100644 --- a/test/Microsoft.ML.Benchmarks/BenchmarkBase.cs +++ b/test/Microsoft.ML.PerformanceTests/BenchmarkBase.cs @@ -8,7 +8,7 @@ using Microsoft.ML.Runtime; using Microsoft.ML.TestFrameworkCommon; -namespace Microsoft.ML.Benchmarks +namespace Microsoft.ML.PerformanceTests { public class BenchmarkBase { diff --git a/test/Microsoft.ML.Benchmarks/CacheDataViewBench.cs b/test/Microsoft.ML.PerformanceTests/CacheDataViewBench.cs similarity index 97% rename from test/Microsoft.ML.Benchmarks/CacheDataViewBench.cs rename to test/Microsoft.ML.PerformanceTests/CacheDataViewBench.cs index bc9a56cfeb..15b2ab6ea6 100644 --- a/test/Microsoft.ML.Benchmarks/CacheDataViewBench.cs +++ b/test/Microsoft.ML.PerformanceTests/CacheDataViewBench.cs @@ -4,10 +4,10 @@ using System; using BenchmarkDotNet.Attributes; -using Microsoft.ML.Benchmarks.Harness; +using Microsoft.ML.PerformanceTests.Harness; using Microsoft.ML.Data; -namespace Microsoft.ML.Benchmarks +namespace Microsoft.ML.PerformanceTests { [CIBenchmark] public class CacheDataViewBench : BenchmarkBase diff --git a/test/Microsoft.ML.Benchmarks/FeaturizeTextBench.cs b/test/Microsoft.ML.PerformanceTests/FeaturizeTextBench.cs similarity index 99% rename from test/Microsoft.ML.Benchmarks/FeaturizeTextBench.cs rename to test/Microsoft.ML.PerformanceTests/FeaturizeTextBench.cs index 46496c5100..fdb7475a8c 100644 --- a/test/Microsoft.ML.Benchmarks/FeaturizeTextBench.cs +++ b/test/Microsoft.ML.PerformanceTests/FeaturizeTextBench.cs @@ -11,7 +11,7 @@ using Microsoft.ML.Transforms.Text; using Xunit; -namespace Microsoft.ML.Benchmarks +namespace Microsoft.ML.PerformanceTests { [Config(typeof(TrainConfig))] public class FeaturizeTextBench : BenchmarkBase diff --git a/test/Microsoft.ML.Benchmarks/Harness/Configs.cs b/test/Microsoft.ML.PerformanceTests/Harness/Configs.cs similarity index 97% rename from test/Microsoft.ML.Benchmarks/Harness/Configs.cs rename to test/Microsoft.ML.PerformanceTests/Harness/Configs.cs index 037df6eefd..07aaee258a 100644 --- a/test/Microsoft.ML.Benchmarks/Harness/Configs.cs +++ b/test/Microsoft.ML.PerformanceTests/Harness/Configs.cs @@ -9,9 +9,9 @@ using BenchmarkDotNet.Toolchains; using BenchmarkDotNet.Toolchains.CsProj; using BenchmarkDotNet.Toolchains.DotNetCli; -using Microsoft.ML.Benchmarks.Harness; +using Microsoft.ML.PerformanceTests.Harness; -namespace Microsoft.ML.Benchmarks +namespace Microsoft.ML.PerformanceTests { public class RecommendedConfig : ManualConfig { diff --git a/test/Microsoft.ML.Benchmarks/Harness/Metrics.cs b/test/Microsoft.ML.PerformanceTests/Harness/Metrics.cs similarity index 98% rename from test/Microsoft.ML.Benchmarks/Harness/Metrics.cs rename to test/Microsoft.ML.PerformanceTests/Harness/Metrics.cs index 5a2da65ec6..1d42b04495 100644 --- a/test/Microsoft.ML.Benchmarks/Harness/Metrics.cs +++ b/test/Microsoft.ML.PerformanceTests/Harness/Metrics.cs @@ -11,7 +11,7 @@ using BenchmarkDotNet.Reports; using BenchmarkDotNet.Running; -namespace Microsoft.ML.Benchmarks +namespace Microsoft.ML.PerformanceTests { public abstract class WithExtraMetrics : BenchmarkBase { diff --git a/test/Microsoft.ML.Benchmarks/Harness/ProjectGenerator.cs b/test/Microsoft.ML.PerformanceTests/Harness/ProjectGenerator.cs similarity index 98% rename from test/Microsoft.ML.Benchmarks/Harness/ProjectGenerator.cs rename to test/Microsoft.ML.PerformanceTests/Harness/ProjectGenerator.cs index 7600f1a7ea..f5a0b96769 100644 --- a/test/Microsoft.ML.Benchmarks/Harness/ProjectGenerator.cs +++ b/test/Microsoft.ML.PerformanceTests/Harness/ProjectGenerator.cs @@ -10,7 +10,7 @@ using BenchmarkDotNet.Toolchains; using BenchmarkDotNet.Toolchains.CsProj; -namespace Microsoft.ML.Benchmarks.Harness +namespace Microsoft.ML.PerformanceTests.Harness { /// /// to avoid side effects of benchmarks affect each other BenchmarkDotNet runs every benchmark in a standalone, dedicated process diff --git a/test/Microsoft.ML.Benchmarks/HashBench.cs b/test/Microsoft.ML.PerformanceTests/HashBench.cs similarity index 99% rename from test/Microsoft.ML.Benchmarks/HashBench.cs rename to test/Microsoft.ML.PerformanceTests/HashBench.cs index a6d62166b9..afb00dc511 100644 --- a/test/Microsoft.ML.Benchmarks/HashBench.cs +++ b/test/Microsoft.ML.PerformanceTests/HashBench.cs @@ -5,12 +5,12 @@ using System; using System.Linq; using BenchmarkDotNet.Attributes; -using Microsoft.ML.Benchmarks.Harness; +using Microsoft.ML.PerformanceTests.Harness; using Microsoft.ML.Data; using Microsoft.ML.Runtime; using Microsoft.ML.Transforms; -namespace Microsoft.ML.Benchmarks +namespace Microsoft.ML.PerformanceTests { [CIBenchmark] public class HashBench : BenchmarkBase diff --git a/test/Microsoft.ML.Benchmarks/Helpers/CIBenchmark.cs b/test/Microsoft.ML.PerformanceTests/Helpers/CIBenchmark.cs similarity index 91% rename from test/Microsoft.ML.Benchmarks/Helpers/CIBenchmark.cs rename to test/Microsoft.ML.PerformanceTests/Helpers/CIBenchmark.cs index 6de9ecb88a..dba140c455 100644 --- a/test/Microsoft.ML.Benchmarks/Helpers/CIBenchmark.cs +++ b/test/Microsoft.ML.PerformanceTests/Helpers/CIBenchmark.cs @@ -3,7 +3,7 @@ // See the LICENSE file in the project root for more information. using System; -namespace Microsoft.ML.Benchmarks.Harness +namespace Microsoft.ML.PerformanceTests.Harness { /// /// This attribute is used to identify the benchmarks diff --git a/test/Microsoft.ML.Benchmarks/Helpers/EmptyWriter.cs b/test/Microsoft.ML.PerformanceTests/Helpers/EmptyWriter.cs similarity index 93% rename from test/Microsoft.ML.Benchmarks/Helpers/EmptyWriter.cs rename to test/Microsoft.ML.PerformanceTests/Helpers/EmptyWriter.cs index a54c8f54df..9298f4261b 100644 --- a/test/Microsoft.ML.Benchmarks/Helpers/EmptyWriter.cs +++ b/test/Microsoft.ML.PerformanceTests/Helpers/EmptyWriter.cs @@ -5,7 +5,7 @@ using System.IO; using System.Text; -namespace Microsoft.ML.Benchmarks +namespace Microsoft.ML.PerformanceTests { // Adding this class to not print anything to the console. // This is required for the current version of BenchmarkDotNet diff --git a/test/Microsoft.ML.Benchmarks/Helpers/EnvironmentFactory.cs b/test/Microsoft.ML.PerformanceTests/Helpers/EnvironmentFactory.cs similarity index 98% rename from test/Microsoft.ML.Benchmarks/Helpers/EnvironmentFactory.cs rename to test/Microsoft.ML.PerformanceTests/Helpers/EnvironmentFactory.cs index d691c70ef9..e7e430a61c 100644 --- a/test/Microsoft.ML.Benchmarks/Helpers/EnvironmentFactory.cs +++ b/test/Microsoft.ML.PerformanceTests/Helpers/EnvironmentFactory.cs @@ -7,7 +7,7 @@ using Microsoft.ML.Trainers; using Microsoft.ML.Transforms; -namespace Microsoft.ML.Benchmarks +namespace Microsoft.ML.PerformanceTests { internal static class EnvironmentFactory { diff --git a/test/Microsoft.ML.Benchmarks/Helpers/Errors.cs b/test/Microsoft.ML.PerformanceTests/Helpers/Errors.cs similarity index 91% rename from test/Microsoft.ML.Benchmarks/Helpers/Errors.cs rename to test/Microsoft.ML.PerformanceTests/Helpers/Errors.cs index 344cbbccb7..94ede8d60a 100644 --- a/test/Microsoft.ML.Benchmarks/Helpers/Errors.cs +++ b/test/Microsoft.ML.PerformanceTests/Helpers/Errors.cs @@ -2,7 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -namespace Microsoft.ML.Benchmarks +namespace Microsoft.ML.PerformanceTests { internal class Errors { diff --git a/test/Microsoft.ML.Benchmarks/Helpers/ExecuteMaml.cs b/test/Microsoft.ML.PerformanceTests/Helpers/ExecuteMaml.cs similarity index 93% rename from test/Microsoft.ML.Benchmarks/Helpers/ExecuteMaml.cs rename to test/Microsoft.ML.PerformanceTests/Helpers/ExecuteMaml.cs index be81171210..e3f075ee61 100644 --- a/test/Microsoft.ML.Benchmarks/Helpers/ExecuteMaml.cs +++ b/test/Microsoft.ML.PerformanceTests/Helpers/ExecuteMaml.cs @@ -4,7 +4,7 @@ using System; using Microsoft.ML.Tools; -namespace Microsoft.ML.Benchmarks +namespace Microsoft.ML.PerformanceTests { internal static class ExecuteMaml { diff --git a/test/Microsoft.ML.Benchmarks/ImageClassificationBench.cs b/test/Microsoft.ML.PerformanceTests/ImageClassificationBench.cs similarity index 99% rename from test/Microsoft.ML.Benchmarks/ImageClassificationBench.cs rename to test/Microsoft.ML.PerformanceTests/ImageClassificationBench.cs index fd7fe0ea44..2b4f267b69 100644 --- a/test/Microsoft.ML.Benchmarks/ImageClassificationBench.cs +++ b/test/Microsoft.ML.PerformanceTests/ImageClassificationBench.cs @@ -15,7 +15,7 @@ using System.Net.Http; using Microsoft.ML.Vision; -namespace Microsoft.ML.Benchmarks +namespace Microsoft.ML.PerformanceTests { [Config(typeof(TrainConfig))] public class ImageClassificationBench : BenchmarkBase diff --git a/test/Microsoft.ML.Benchmarks/KMeansAndLogisticRegressionBench.cs b/test/Microsoft.ML.PerformanceTests/KMeansAndLogisticRegressionBench.cs similarity index 96% rename from test/Microsoft.ML.Benchmarks/KMeansAndLogisticRegressionBench.cs rename to test/Microsoft.ML.PerformanceTests/KMeansAndLogisticRegressionBench.cs index 7157963c24..82c0da7022 100644 --- a/test/Microsoft.ML.Benchmarks/KMeansAndLogisticRegressionBench.cs +++ b/test/Microsoft.ML.PerformanceTests/KMeansAndLogisticRegressionBench.cs @@ -3,12 +3,12 @@ // See the LICENSE file in the project root for more information. using BenchmarkDotNet.Attributes; -using Microsoft.ML.Benchmarks.Harness; +using Microsoft.ML.PerformanceTests.Harness; using Microsoft.ML.Calibrators; using Microsoft.ML.Data; using Microsoft.ML.Trainers; -namespace Microsoft.ML.Benchmarks +namespace Microsoft.ML.PerformanceTests { [CIBenchmark] public class KMeansAndLogisticRegressionBench : BenchmarkBase diff --git a/test/Microsoft.ML.Benchmarks/Microsoft.ML.Benchmarks.csproj b/test/Microsoft.ML.PerformanceTests/Microsoft.ML.PerformanceTests.csproj similarity index 67% rename from test/Microsoft.ML.Benchmarks/Microsoft.ML.Benchmarks.csproj rename to test/Microsoft.ML.PerformanceTests/Microsoft.ML.PerformanceTests.csproj index af434d9820..217d61ed05 100644 --- a/test/Microsoft.ML.Benchmarks/Microsoft.ML.Benchmarks.csproj +++ b/test/Microsoft.ML.PerformanceTests/Microsoft.ML.PerformanceTests.csproj @@ -2,6 +2,8 @@ Exe false + false + true @@ -30,4 +32,20 @@ + + + https://aka.ms/mlnet-resources/meta/%(Identity) + $([System.IO.Path]::GetTempPath())/MLNET/ + + + + + + + + diff --git a/test/Microsoft.ML.Benchmarks/Numeric/Ranking.cs b/test/Microsoft.ML.PerformanceTests/Numeric/Ranking.cs similarity index 99% rename from test/Microsoft.ML.Benchmarks/Numeric/Ranking.cs rename to test/Microsoft.ML.PerformanceTests/Numeric/Ranking.cs index c737c95b5d..78504fe752 100644 --- a/test/Microsoft.ML.Benchmarks/Numeric/Ranking.cs +++ b/test/Microsoft.ML.PerformanceTests/Numeric/Ranking.cs @@ -11,7 +11,7 @@ using Microsoft.ML.Transforms; using Microsoft.ML.TestFrameworkCommon; -namespace Microsoft.ML.Benchmarks +namespace Microsoft.ML.PerformanceTests { [Config(typeof(TrainConfig))] public class RankingTrain : BenchmarkBase diff --git a/test/Microsoft.ML.Benchmarks/PredictionEngineBench.cs b/test/Microsoft.ML.PerformanceTests/PredictionEngineBench.cs similarity index 98% rename from test/Microsoft.ML.Benchmarks/PredictionEngineBench.cs rename to test/Microsoft.ML.PerformanceTests/PredictionEngineBench.cs index 9255f14c52..0e2f2823f3 100644 --- a/test/Microsoft.ML.Benchmarks/PredictionEngineBench.cs +++ b/test/Microsoft.ML.PerformanceTests/PredictionEngineBench.cs @@ -3,12 +3,12 @@ // See the LICENSE file in the project root for more information. using BenchmarkDotNet.Attributes; -using Microsoft.ML.Benchmarks.Harness; +using Microsoft.ML.PerformanceTests.Harness; using Microsoft.ML.Data; using Microsoft.ML.Trainers; using Microsoft.ML.Transforms; -namespace Microsoft.ML.Benchmarks +namespace Microsoft.ML.PerformanceTests { [CIBenchmark] public class PredictionEngineBench : BenchmarkBase diff --git a/test/Microsoft.ML.PerformanceTests/Program.cs b/test/Microsoft.ML.PerformanceTests/Program.cs new file mode 100644 index 0000000000..a5f2824815 --- /dev/null +++ b/test/Microsoft.ML.PerformanceTests/Program.cs @@ -0,0 +1,53 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Globalization; +using System.Threading; +using BenchmarkDotNet.Running; + +namespace Microsoft.ML.PerformanceTests +{ + class Program + { + /// + /// execute dotnet run -c Release and choose the benchmarks you want to run + /// + /// + static void Main(string[] args) + { + // enforce Neutral Language as "en-us" because the input data files use dot as decimal separator (and it fails for cultures with ",") + Thread.CurrentThread.CurrentCulture = CultureInfo.InvariantCulture; + + /* + Available Benchmarks: + #0 TextPredictionEngineCreationBenchmark + #1 CacheDataViewBench + #2 FeaturizeTextBench + #3 HashBench + #4 ImageClassificationBench + #5 KMeansAndLogisticRegressionBench + #6 MulticlassClassificationTest + #7 MulticlassClassificationTrain + #8 PredictionEngineBench + #9 RankingTest + #10 RankingTrain + #11 RffTransformTrain + #12 ShuffleRowsBench + #13 StochasticDualCoordinateAscentClassifierBench + #14 TextLoaderBench + */ + + // TO-DO: Further investigate how to run each benchmark separately. + // Arcade's `-performanceTest` command results in a predefined dotnet call that does not allow for additional arguments + // to be passed (say, for selecting individual benchmarks to run). + // Link to code: + // https://github.com/dotnet/arcade/blob/4873d157a8f34f8cc7e28b3f9938b32c642ef542/src/Microsoft.DotNet.Arcade.Sdk/tools/Performance.targets#L16-L19 + BenchmarkSwitcher + .FromAssembly(typeof(Program).Assembly) + .RunAll(new RecommendedConfig()); + + } + } +} diff --git a/test/Microsoft.ML.Benchmarks/README.md b/test/Microsoft.ML.PerformanceTests/README.md similarity index 63% rename from test/Microsoft.ML.Benchmarks/README.md rename to test/Microsoft.ML.PerformanceTests/README.md index 368c00030e..57807b6442 100644 --- a/test/Microsoft.ML.Benchmarks/README.md +++ b/test/Microsoft.ML.PerformanceTests/README.md @@ -1,4 +1,4 @@ -# ML.NET Benchmarks +# ML.NET Benchmarks/Performance Tests This project contains performance benchmarks. @@ -8,46 +8,19 @@ This project contains performance benchmarks. git submodule update --init -**Pre-requisite:** On a clean repo with initialized submodules, `build.cmd` at the root installs the right version of dotnet.exe and builds the solution. You need to build the solution in `Release` with native dependencies. +**Pre-requisite:** On a clean repo with initialized submodules, `build.cmd` at the root installs the right version of dotnet.exe and builds the solution. You need to build the solution in `Release`. - build.cmd -release -buildNative - -Moreover, to run some of the benchmarks you have to download external dependencies. + build.cmd -configuration Release - build.cmd -- /t:DownloadExternalTestFiles /p:IncludeBenchmarkData=true +1. Navigate to the performance tests directory (machinelearning\test\Microsoft.ML.PerformanceTests) -1. Navigate to the benchmarks directory (machinelearning\test\Microsoft.ML.Benchmarks) - -2. Run the benchmarks in Release, choose one of the benchmarks when prompted - -```log - dotnet run -c Release -``` - -3. To run specific tests only, pass in the filter to the harness: - -```log - dotnet run -c Release -- --filter namespace* - dotnet run -c Release -- --filter *typeName* - dotnet run -c Release -- --filter *.methodName - dotnet run -c Release -- --filter namespace.typeName.methodName -``` - -4. GC Statistics - -To get the total number of allocated managed memory please pass additional console argument: `--memory` or just `-m`. This feature is disabled by default because it requires an additional iteration (which is expensive for time consuming benchmarks). - -| Gen 0 | Gen 1 | Gen 2 | Allocated | -|------------:|-----------:|----------:|----------:| -| 175000.0000 | 33000.0000 | 7000.0000 | 238.26 MB | - -5. To find out more about supported command line arguments run +2. Run the benchmarks in Release: ```log - dotnet run -c Release -- --help + build.cmd -configuration Release -performanceTest ``` -## .NET Core 3.0 +## .NET Core 3.1 **Pre-requisite:** Follow the [netcoreapp3.1 instructions](../../docs/building/netcoreapp3.1-instructions.md). @@ -55,12 +28,12 @@ To get the total number of allocated managed memory please pass additional conso $env:DOTNET_MULTILEVEL_LOOKUP=0 -1. Navigate to the benchmarks directory (machinelearning\test\Microsoft.ML.Benchmarks) +1. Navigate to the benchmarks directory (machinelearning\test\Microsoft.ML.PerformanceTests) -2. Run the benchmarks in `Release-netcoreapp3_1` configuration, choose one of the benchmarks when prompted +2. Run the benchmarks in `Release-netcoreapp3_1` configuration: ```log - ..\..\Tools\dotnetcli\dotnet.exe run -c Release-netcoreapp3_1 + build.cmd -configuration Release-netcoreapp3_1 -performanceTest ``` ## Authoring new benchmarks @@ -91,7 +64,7 @@ you can debug this test locally by: 1- Building the solution in the release mode locally -build.cmd -release -buildNative +build.cmd -configuration Release -performanceTest 2- Changing the configuration in Visual Studio from Debug -> Release 3- Changing the annotation in the `BenchmarksProjectIsNotBroken` to replace `BenchmarkTheory` with `Theory`, as below. diff --git a/test/Microsoft.ML.Benchmarks/RffTransform.cs b/test/Microsoft.ML.PerformanceTests/RffTransform.cs similarity index 96% rename from test/Microsoft.ML.Benchmarks/RffTransform.cs rename to test/Microsoft.ML.PerformanceTests/RffTransform.cs index 659f87c2c4..81f6507b95 100644 --- a/test/Microsoft.ML.Benchmarks/RffTransform.cs +++ b/test/Microsoft.ML.PerformanceTests/RffTransform.cs @@ -4,12 +4,12 @@ using System.IO; using BenchmarkDotNet.Attributes; -using Microsoft.ML.Benchmarks.Harness; +using Microsoft.ML.PerformanceTests.Harness; using Microsoft.ML.Data; using Microsoft.ML.TestFrameworkCommon; using Microsoft.ML.Transforms; -namespace Microsoft.ML.Benchmarks +namespace Microsoft.ML.PerformanceTests { [CIBenchmark] public class RffTransformTrain : BenchmarkBase diff --git a/test/Microsoft.ML.Benchmarks/ShuffleRowsBench.cs b/test/Microsoft.ML.PerformanceTests/ShuffleRowsBench.cs similarity index 94% rename from test/Microsoft.ML.Benchmarks/ShuffleRowsBench.cs rename to test/Microsoft.ML.PerformanceTests/ShuffleRowsBench.cs index 95ff4be7e3..710923bad6 100644 --- a/test/Microsoft.ML.Benchmarks/ShuffleRowsBench.cs +++ b/test/Microsoft.ML.PerformanceTests/ShuffleRowsBench.cs @@ -3,10 +3,10 @@ // See the LICENSE file in the project root for more information. using BenchmarkDotNet.Attributes; -using Microsoft.ML.Benchmarks.Harness; +using Microsoft.ML.PerformanceTests.Harness; using Microsoft.ML.Data; -namespace Microsoft.ML.Benchmarks +namespace Microsoft.ML.PerformanceTests { [CIBenchmark] public class ShuffleRowsBench : BenchmarkBase diff --git a/test/Microsoft.ML.Benchmarks/StochasticDualCoordinateAscentClassifierBench.cs b/test/Microsoft.ML.PerformanceTests/StochasticDualCoordinateAscentClassifierBench.cs similarity index 98% rename from test/Microsoft.ML.Benchmarks/StochasticDualCoordinateAscentClassifierBench.cs rename to test/Microsoft.ML.PerformanceTests/StochasticDualCoordinateAscentClassifierBench.cs index 8aaeb53711..6908af3c01 100644 --- a/test/Microsoft.ML.Benchmarks/StochasticDualCoordinateAscentClassifierBench.cs +++ b/test/Microsoft.ML.PerformanceTests/StochasticDualCoordinateAscentClassifierBench.cs @@ -6,13 +6,13 @@ using System.Globalization; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Engines; -using Microsoft.ML.Benchmarks.Harness; +using Microsoft.ML.PerformanceTests.Harness; using Microsoft.ML.Data; using Microsoft.ML.Trainers; using Microsoft.ML.Transforms; using Microsoft.ML.Transforms.Text; -namespace Microsoft.ML.Benchmarks +namespace Microsoft.ML.PerformanceTests { [CIBenchmark] public class StochasticDualCoordinateAscentClassifierBench : WithExtraMetrics diff --git a/test/Microsoft.ML.Benchmarks/Text/MultiClassClassification.cs b/test/Microsoft.ML.PerformanceTests/Text/MultiClassClassification.cs similarity index 99% rename from test/Microsoft.ML.Benchmarks/Text/MultiClassClassification.cs rename to test/Microsoft.ML.PerformanceTests/Text/MultiClassClassification.cs index ade958e2e0..87234138c2 100644 --- a/test/Microsoft.ML.Benchmarks/Text/MultiClassClassification.cs +++ b/test/Microsoft.ML.PerformanceTests/Text/MultiClassClassification.cs @@ -10,7 +10,7 @@ using Microsoft.ML.Transforms; using Microsoft.ML.TestFrameworkCommon; -namespace Microsoft.ML.Benchmarks +namespace Microsoft.ML.PerformanceTests { [Config(typeof(TrainConfig))] public class MulticlassClassificationTrain : BenchmarkBase diff --git a/test/Microsoft.ML.Benchmarks/TextLoaderBench.cs b/test/Microsoft.ML.PerformanceTests/TextLoaderBench.cs similarity index 99% rename from test/Microsoft.ML.Benchmarks/TextLoaderBench.cs rename to test/Microsoft.ML.PerformanceTests/TextLoaderBench.cs index 8f1faf6b76..73abceed60 100644 --- a/test/Microsoft.ML.Benchmarks/TextLoaderBench.cs +++ b/test/Microsoft.ML.PerformanceTests/TextLoaderBench.cs @@ -11,7 +11,7 @@ using Microsoft.ML.Transforms.Text; using Xunit; -namespace Microsoft.ML.Benchmarks +namespace Microsoft.ML.PerformanceTests { [Config(typeof(TrainConfig))] public class TextLoaderBench : BenchmarkBase diff --git a/test/Microsoft.ML.Benchmarks/TextPredictionEngineCreation.cs b/test/Microsoft.ML.PerformanceTests/TextPredictionEngineCreation.cs similarity index 98% rename from test/Microsoft.ML.Benchmarks/TextPredictionEngineCreation.cs rename to test/Microsoft.ML.PerformanceTests/TextPredictionEngineCreation.cs index 81eedd5651..91e1019273 100644 --- a/test/Microsoft.ML.Benchmarks/TextPredictionEngineCreation.cs +++ b/test/Microsoft.ML.PerformanceTests/TextPredictionEngineCreation.cs @@ -5,7 +5,7 @@ using System.IO; using BenchmarkDotNet.Attributes; using Microsoft.ML; -using Microsoft.ML.Benchmarks; +using Microsoft.ML.PerformanceTests; using Microsoft.ML.Trainers; namespace micro