Added Linux & Mac changes for Arcade (#5479)

mstfbl · mstfbl · commit bf9701d24663 · 2020-11-10T13:10:03.000-08:00
* Initial Windows, Linux, Macos builds test

* Add Linux/MacOS specific CI requirements

* Run Arcade CI tests on MacOS/Linux

* Fix final package building

* Add benchmark download to benchmars .csporj file

* Print detailed status of each unit test

* Install CentOS &amp; Ubuntu build dependencies

* Use container names to differenciate between Ubuntu &amp; CentOS

* Remove sudo usage in CentOS

* Fix Linux build dependencies

* Add -y param to apt install

* Remove installation of Linux dependencies

* Minor additions

* Rename Benchmarks to PerformanceTests for Arcade

* Changes

* Added benchmark doc changes

* Pre-merge changes
diff --git a/.gitignore b/.gitignore
@@ -6,6 +6,7 @@
 # Tool Runtime Dir
 /[Tt]ools/
 /.dotnet/
+/.packages/
 
 # User-specific files
 *.suo
diff --git a/.vsts-dotnet-ci.yml b/.vsts-dotnet-ci.yml
@@ -11,42 +11,42 @@ resources:
     image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-mlnet-20200515184230-2c829e8
 
 jobs:
-# - template: /build/ci/job-template.yml
-#   parameters:
-#     name: Centos_x64_NetCoreApp31
-#     buildScript: ./build.sh
-#     container: CentosContainer
-#     customMatrixes:
-#       Debug_Build:
-#         _configuration: Debug-netcoreapp3_1
-#         _config_short: DI
-#         _includeBenchmarkData: false
-#         _targetFramework: netcoreapp3.1
-#       Release_Build:
-#         _configuration: Release-netcoreapp3_1
-#         _config_short: RI
-#         _includeBenchmarkData: true
-#         _targetFramework: netcoreapp3.1
-#     innerLoop: true
-#     pool:
-#       name:  Hosted Ubuntu 1604
+- template: /build/ci/job-template.yml
+  parameters:
+    name: Centos_x64_NetCoreApp31
+    buildScript: ./build.sh
+    container: CentosContainer
+    customMatrixes:
+      Debug_Build:
+        _configuration: Debug-netcoreapp3_1
+        _config_short: DI
+        _includeBenchmarkData: false
+        _targetFramework: netcoreapp3.1
+      Release_Build:
+        _configuration: Release-netcoreapp3_1
+        _config_short: RI
+        _includeBenchmarkData: true
+        _targetFramework: netcoreapp3.1
+    innerLoop: true
+    pool:
+      name: Hosted Ubuntu 1604
 
-# - template: /build/ci/job-template.yml
-#   parameters:
-#     name: Ubuntu_x64_NetCoreApp21
-#     buildScript: ./build.sh
-#     container: UbuntuContainer
-#     innerLoop: true
-#     pool:
-#       name:  Hosted Ubuntu 1604
+- template: /build/ci/job-template.yml
+  parameters:
+    name: Ubuntu_x64_NetCoreApp21
+    buildScript: ./build.sh
+    container: UbuntuContainer
+    innerLoop: true
+    pool:
+      name: Hosted Ubuntu 1604
 
-# - template: /build/ci/job-template.yml
-#   parameters:
-#     name: MacOS_x64_NetCoreApp21
-#     buildScript: ./build.sh
-#     innerLoop: true
-#     pool:
-#       name: Hosted macOS
+- template: /build/ci/job-template.yml
+  parameters:
+    name: MacOS_x64_NetCoreApp21
+    buildScript: ./build.sh
+    innerLoop: true
+    pool:
+      name: Hosted macOS
 
 - template: /build/ci/job-template.yml
   parameters:
diff --git a/Microsoft.ML.sln b/Microsoft.ML.sln
@@ -48,7 +48,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "pkg", "pkg", "{D3D38B03-B55
 		pkg\Directory.Build.props = pkg\Directory.Build.props
 	EndProjectSection
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Benchmarks", "test\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj", "{7A9DB75F-2CA5-4184-9EF5-1F17EB39483F}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.PerformanceTests", "test\Microsoft.ML.PerformanceTests\Microsoft.ML.PerformanceTests.csproj", "{7A9DB75F-2CA5-4184-9EF5-1F17EB39483F}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Maml", "src\Microsoft.ML.Maml\Microsoft.ML.Maml.csproj", "{64F40A0D-D4C2-4AA7-8470-E9CC437827E4}"
 EndProject
diff --git a/build.sh b/build.sh
@@ -10,4 +10,4 @@ while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symli
 done
 DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
 
-"$DIR/eng/common/build.sh" --restore --build --pack --warnAsError false "$@"
+"$DIR/eng/common/build.sh" --restore --build --warnAsError false "$@"
diff --git a/build/ci/job-template.yml b/build/ci/job-template.yml
@@ -58,9 +58,10 @@ jobs:
           brew untap local/openssl |
           brew untap local/python2
         displayName: MacOS Homebrew bug Workaround
+    # Extra MacOS step required to install OS-specific dependencies
     - ${{ if eq(parameters.pool.name, 'Hosted macOS') }}:
       - script: brew update && brew unlink python@3.8 && brew install mono-libgdiplus && brew install $(Build.SourcesDirectory)/build/libomp.rb && brew link libomp --force
-        displayName: Install build dependencies
+        displayName: Install MacOS build dependencies
     - ${{ if and( eq(parameters.nightlyBuild, 'true'), eq(parameters.pool.name, 'Hosted Ubuntu 1604')) }}:
       - bash: echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$(nightlyBuildRunPath):$LD_LIBRARY_PATH"
         displayName: Set LD_LIBRARY_PATH for Ubuntu and CentOS to locate Native shared library in current running path
@@ -186,5 +187,5 @@ jobs:
         artifactName: ${{ parameters.name }} $(_config_short)
         artifactType: container
     - ${{ if eq(parameters.nightlyBuild, 'false') }}:
-      - script: ${{ parameters.buildScript }} -pack  -ci
+      - script: ${{ parameters.buildScript }} -pack -ci
         displayName: Build Packages
diff --git a/src/Microsoft.ML.Core/Properties/AssemblyInfo.cs b/src/Microsoft.ML.Core/Properties/AssemblyInfo.cs
@@ -13,7 +13,7 @@
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.InferenceTesting" + PublicKey.Value)]
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.OnnxTransformerTest" + PublicKey.Value)]
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.AutoML.Tests" + PublicKey.Value)]
-[assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Benchmarks" + PublicKey.Value)]
+[assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.PerformanceTests" + PublicKey.Value)]
 
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.EntryPoints" + PublicKey.Value)]
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Maml" + PublicKey.Value)]
diff --git a/src/Microsoft.ML.CpuMath/AssemblyInfo.cs b/src/Microsoft.ML.CpuMath/AssemblyInfo.cs
@@ -15,6 +15,6 @@
 [assembly: InternalsVisibleTo("Microsoft.ML.Sweeper" + PublicKey.Value)]
 [assembly: InternalsVisibleTo("Microsoft.ML.TimeSeries" + PublicKey.Value)]
 [assembly: InternalsVisibleTo("Microsoft.ML.Transforms" + PublicKey.Value)]
-[assembly: InternalsVisibleTo("Microsoft.ML.Benchmarks.Tests" + PublicKey.Value)]
+[assembly: InternalsVisibleTo("Microsoft.ML.PerformanceTests.Tests" + PublicKey.Value)]
 
 [assembly: WantsToBeBestFriends]
diff --git a/src/Microsoft.ML.Data/Properties/AssemblyInfo.cs b/src/Microsoft.ML.Data/Properties/AssemblyInfo.cs
@@ -12,10 +12,10 @@
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.OnnxTransformerTest" + PublicKey.Value)]
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Predictor.Tests" + PublicKey.Value)]
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.TimeSeries.Tests" + PublicKey.Value)]
-[assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Benchmarks" + PublicKey.Value)]
+[assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.PerformanceTests" + PublicKey.Value)]
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.AutoML.Tests" + PublicKey.Value)]
 
-[assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Benchmarks" + PublicKey.Value)]
+[assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.PerformanceTests" + PublicKey.Value)]
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.EntryPoints" + PublicKey.Value)]
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Maml" + PublicKey.Value)]
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.ResultProcessor" + PublicKey.Value)]
diff --git a/src/Microsoft.ML.Maml/Properties/AssemblyInfo.cs b/src/Microsoft.ML.Maml/Properties/AssemblyInfo.cs
@@ -6,7 +6,7 @@
 using Microsoft.ML;
 
 [assembly: InternalsVisibleTo("Microsoft.ML.TestFramework" + PublicKey.Value)]
-[assembly: InternalsVisibleTo("Microsoft.ML.Benchmarks" + PublicKey.Value)]
+[assembly: InternalsVisibleTo("Microsoft.ML.PerformanceTests" + PublicKey.Value)]
 [assembly: InternalsVisibleTo("Microsoft.ML.ResultProcessor" + PublicKey.Value)]
 
 [assembly: InternalsVisibleTo("RunTests" + InternalPublicKey.Value)]
diff --git a/src/Redist/Microsoft.ML.DnnImageFeaturizer.ModelRedist/Microsoft.ML.DnnImageFeaturizer.ModelRedist.proj b/src/Redist/Microsoft.ML.DnnImageFeaturizer.ModelRedist/Microsoft.ML.DnnImageFeaturizer.ModelRedist.proj
@@ -123,4 +123,6 @@
 
   <Target Name="Test" />
 
+  <Target Name="PerformanceTest"/>
+
 </Project>
diff --git a/test/Microsoft.ML.Benchmarks.Tests/BenchmarksTest.cs b/test/Microsoft.ML.Benchmarks.Tests/BenchmarksTest.cs
@@ -10,13 +10,13 @@
 using BenchmarkDotNet.Jobs;
 using BenchmarkDotNet.Loggers;
 using BenchmarkDotNet.Running;
-using Microsoft.ML.Benchmarks.Harness;
+using Microsoft.ML.PerformanceTests.Harness;
 using Microsoft.ML.TestFramework;
 using Microsoft.ML.TestFramework.Attributes;
 using Xunit;
 using Xunit.Abstractions;
 
-namespace Microsoft.ML.Benchmarks.Tests
+namespace Microsoft.ML.PerformanceTests.Tests
 {
     public class TestConfig : RecommendedConfig
     {
diff --git a/test/Microsoft.ML.Benchmarks.Tests/Microsoft.ML.Benchmarks.Tests.csproj b/test/Microsoft.ML.Benchmarks.Tests/Microsoft.ML.Benchmarks.Tests.csproj
@@ -1,7 +1,7 @@
 ﻿<Project Sdk="Microsoft.NET.Sdk">
 
   <ItemGroup>
-    <ProjectReference Include="..\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj" />
+    <ProjectReference Include="..\Microsoft.ML.PerformanceTests\Microsoft.ML.PerformanceTests.csproj" />
 
     <NativeAssemblyReference Include="CpuMathNative" />
     <NativeAssemblyReference Include="MklImports" />
diff --git a/test/Microsoft.ML.Benchmarks/Program.cs b/test/Microsoft.ML.Benchmarks/Program.cs
diff --git a/test/Microsoft.ML.CpuMath.PerformanceTests/Microsoft.ML.CpuMath.PerformanceTests.csproj b/test/Microsoft.ML.CpuMath.PerformanceTests/Microsoft.ML.CpuMath.PerformanceTests.csproj
@@ -10,6 +10,8 @@
     -->
     <RuntimeIdentifier></RuntimeIdentifier>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+    <IsUnitTestProject>false</IsUnitTestProject>
+    <IsPerformanceTestProject>true</IsPerformanceTestProject>
   </PropertyGroup>
   
   <ItemGroup>
diff --git a/test/Microsoft.ML.PerformanceTests/BenchmarkBase.cs b/test/Microsoft.ML.PerformanceTests/BenchmarkBase.cs
@@ -8,7 +8,7 @@
 using Microsoft.ML.Runtime;
 using Microsoft.ML.TestFrameworkCommon;
 
-namespace Microsoft.ML.Benchmarks
+namespace Microsoft.ML.PerformanceTests
 {
     public class BenchmarkBase
     {
diff --git a/test/Microsoft.ML.PerformanceTests/CacheDataViewBench.cs b/test/Microsoft.ML.PerformanceTests/CacheDataViewBench.cs
@@ -4,10 +4,10 @@
 
 using System;
 using BenchmarkDotNet.Attributes;
-using Microsoft.ML.Benchmarks.Harness;
+using Microsoft.ML.PerformanceTests.Harness;
 using Microsoft.ML.Data;
 
-namespace Microsoft.ML.Benchmarks
+namespace Microsoft.ML.PerformanceTests
 {
     [CIBenchmark]
     public class CacheDataViewBench : BenchmarkBase
diff --git a/test/Microsoft.ML.PerformanceTests/FeaturizeTextBench.cs b/test/Microsoft.ML.PerformanceTests/FeaturizeTextBench.cs
@@ -11,7 +11,7 @@
 using Microsoft.ML.Transforms.Text;
 using Xunit;
 
-namespace Microsoft.ML.Benchmarks
+namespace Microsoft.ML.PerformanceTests
 {
     [Config(typeof(TrainConfig))]
     public class FeaturizeTextBench : BenchmarkBase
diff --git a/test/Microsoft.ML.PerformanceTests/Harness/Configs.cs b/test/Microsoft.ML.PerformanceTests/Harness/Configs.cs
@@ -9,9 +9,9 @@
 using BenchmarkDotNet.Toolchains;
 using BenchmarkDotNet.Toolchains.CsProj;
 using BenchmarkDotNet.Toolchains.DotNetCli;
-using Microsoft.ML.Benchmarks.Harness;
+using Microsoft.ML.PerformanceTests.Harness;
 
-namespace Microsoft.ML.Benchmarks
+namespace Microsoft.ML.PerformanceTests
 {
     public class RecommendedConfig : ManualConfig
     {
diff --git a/test/Microsoft.ML.PerformanceTests/Harness/Metrics.cs b/test/Microsoft.ML.PerformanceTests/Harness/Metrics.cs
@@ -11,7 +11,7 @@
 using BenchmarkDotNet.Reports;
 using BenchmarkDotNet.Running;
 
-namespace Microsoft.ML.Benchmarks
+namespace Microsoft.ML.PerformanceTests
 {
     public abstract class WithExtraMetrics : BenchmarkBase
     {
diff --git a/test/Microsoft.ML.PerformanceTests/Harness/ProjectGenerator.cs b/test/Microsoft.ML.PerformanceTests/Harness/ProjectGenerator.cs
@@ -10,7 +10,7 @@
 using BenchmarkDotNet.Toolchains;
 using BenchmarkDotNet.Toolchains.CsProj;
 
-namespace Microsoft.ML.Benchmarks.Harness
+namespace Microsoft.ML.PerformanceTests.Harness
 {
     /// <summary>
     /// to avoid side effects of benchmarks affect each other BenchmarkDotNet runs every benchmark in a standalone, dedicated process
diff --git a/test/Microsoft.ML.PerformanceTests/HashBench.cs b/test/Microsoft.ML.PerformanceTests/HashBench.cs
@@ -5,12 +5,12 @@
 using System;
 using System.Linq;
 using BenchmarkDotNet.Attributes;
-using Microsoft.ML.Benchmarks.Harness;
+using Microsoft.ML.PerformanceTests.Harness;
 using Microsoft.ML.Data;
 using Microsoft.ML.Runtime;
 using Microsoft.ML.Transforms;
 
-namespace Microsoft.ML.Benchmarks
+namespace Microsoft.ML.PerformanceTests
 {
     [CIBenchmark]
     public class HashBench : BenchmarkBase
diff --git a/test/Microsoft.ML.PerformanceTests/Helpers/CIBenchmark.cs b/test/Microsoft.ML.PerformanceTests/Helpers/CIBenchmark.cs
@@ -3,7 +3,7 @@
 // See the LICENSE file in the project root for more information.
 
 using System;
-namespace Microsoft.ML.Benchmarks.Harness
+namespace Microsoft.ML.PerformanceTests.Harness
 {
     /// <summary>
     /// This attribute is used to identify the benchmarks 
diff --git a/test/Microsoft.ML.PerformanceTests/Helpers/EmptyWriter.cs b/test/Microsoft.ML.PerformanceTests/Helpers/EmptyWriter.cs
@@ -5,7 +5,7 @@
 using System.IO;
 using System.Text;
 
-namespace Microsoft.ML.Benchmarks
+namespace Microsoft.ML.PerformanceTests
 {
     // Adding this class to not print anything to the console.
     // This is required for the current version of BenchmarkDotNet
diff --git a/test/Microsoft.ML.PerformanceTests/Helpers/EnvironmentFactory.cs b/test/Microsoft.ML.PerformanceTests/Helpers/EnvironmentFactory.cs
@@ -7,7 +7,7 @@
 using Microsoft.ML.Trainers;
 using Microsoft.ML.Transforms;
 
-namespace Microsoft.ML.Benchmarks
+namespace Microsoft.ML.PerformanceTests
 {
     internal static class EnvironmentFactory
     {
diff --git a/test/Microsoft.ML.PerformanceTests/Helpers/Errors.cs b/test/Microsoft.ML.PerformanceTests/Helpers/Errors.cs
@@ -2,7 +2,7 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-namespace Microsoft.ML.Benchmarks
+namespace Microsoft.ML.PerformanceTests
 {
     internal class Errors
     {
diff --git a/test/Microsoft.ML.PerformanceTests/Helpers/ExecuteMaml.cs b/test/Microsoft.ML.PerformanceTests/Helpers/ExecuteMaml.cs
@@ -4,7 +4,7 @@
 
 using System;
 using Microsoft.ML.Tools;
-namespace Microsoft.ML.Benchmarks
+namespace Microsoft.ML.PerformanceTests
 {
     internal static class ExecuteMaml
     {
diff --git a/test/Microsoft.ML.PerformanceTests/ImageClassificationBench.cs b/test/Microsoft.ML.PerformanceTests/ImageClassificationBench.cs
@@ -15,7 +15,7 @@
 using System.Net.Http;
 using Microsoft.ML.Vision;
 
-namespace Microsoft.ML.Benchmarks
+namespace Microsoft.ML.PerformanceTests
 {
     [Config(typeof(TrainConfig))]
     public class ImageClassificationBench : BenchmarkBase
diff --git a/test/Microsoft.ML.PerformanceTests/KMeansAndLogisticRegressionBench.cs b/test/Microsoft.ML.PerformanceTests/KMeansAndLogisticRegressionBench.cs
@@ -3,12 +3,12 @@
 // See the LICENSE file in the project root for more information.
 
 using BenchmarkDotNet.Attributes;
-using Microsoft.ML.Benchmarks.Harness;
+using Microsoft.ML.PerformanceTests.Harness;
 using Microsoft.ML.Calibrators;
 using Microsoft.ML.Data;
 using Microsoft.ML.Trainers;
 
-namespace Microsoft.ML.Benchmarks
+namespace Microsoft.ML.PerformanceTests
 {
     [CIBenchmark]
     public class KMeansAndLogisticRegressionBench : BenchmarkBase
diff --git a/test/Microsoft.ML.PerformanceTests/Microsoft.ML.PerformanceTests.csproj b/test/Microsoft.ML.PerformanceTests/Microsoft.ML.PerformanceTests.csproj
diff --git a/test/Microsoft.ML.PerformanceTests/Numeric/Ranking.cs b/test/Microsoft.ML.PerformanceTests/Numeric/Ranking.cs
diff --git a/test/Microsoft.ML.PerformanceTests/PredictionEngineBench.cs b/test/Microsoft.ML.PerformanceTests/PredictionEngineBench.cs
diff --git a/test/Microsoft.ML.PerformanceTests/Program.cs b/test/Microsoft.ML.PerformanceTests/Program.cs
diff --git a/test/Microsoft.ML.PerformanceTests/README.md b/test/Microsoft.ML.PerformanceTests/README.md
diff --git a/test/Microsoft.ML.PerformanceTests/RffTransform.cs b/test/Microsoft.ML.PerformanceTests/RffTransform.cs
diff --git a/test/Microsoft.ML.PerformanceTests/ShuffleRowsBench.cs b/test/Microsoft.ML.PerformanceTests/ShuffleRowsBench.cs
diff --git a/test/Microsoft.ML.PerformanceTests/StochasticDualCoordinateAscentClassifierBench.cs b/test/Microsoft.ML.PerformanceTests/StochasticDualCoordinateAscentClassifierBench.cs
diff --git a/test/Microsoft.ML.PerformanceTests/Text/MultiClassClassification.cs b/test/Microsoft.ML.PerformanceTests/Text/MultiClassClassification.cs
diff --git a/test/Microsoft.ML.PerformanceTests/TextLoaderBench.cs b/test/Microsoft.ML.PerformanceTests/TextLoaderBench.cs
diff --git a/test/Microsoft.ML.PerformanceTests/TextPredictionEngineCreation.cs b/test/Microsoft.ML.PerformanceTests/TextPredictionEngineCreation.cs
