ML.NET rolling and fixed version CI legs

NuGet.config

@@ -8,5 +8,6 @@
     <add key="dotnet-core" value="https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json" />
     <add key="api.nuget.org" value="https://api.nuget.org/v3/index.json" />
     <add key="aspnet-aspnetcore" value="https://dotnetfeed.blob.core.windows.net/aspnet-aspnetcore/index.json" />
+    <add key="mldotnet" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/MachineLearning/nuget/v3/index.json" />
   </packageSources>
 </configuration>

azure-pipelines.yml

@@ -108,7 +108,7 @@ jobs:
       frameworks: # for public jobs we want to make sure that the PRs don't break x86
         - netcoreapp5.0
 
-  # Windows x64 ML.NET benchmarks
+  # Windows x64 ML.NET benchmarks: fixed ML.NET, rolling .NET Core
   - template: /eng/performance/benchmark_jobs.yml
     parameters:
       osName: windows
@@ -119,10 +119,26 @@ jobs:
       queue: Windows.10.Amd64.ClientRS5.Open
       csproj: src\benchmarks\real-world\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj
       runCategories: 'mldotnet'
-      frameworks: # for ML.NET jobs we want to check .NET Core 3.0 and 5.0 only
+      libraryVersion: fixed
+      frameworks: # for fixed ML.NET version we want to check latest .NET Core bits (to detect regressions in .NET Core)
         - netcoreapp5.0
         - netcoreapp3.1
 
+  # Windows x64 ML.NET benchmarks: rolling ML.NET, fixed .NET Core
+  - template: /eng/performance/benchmark_jobs.yml
+    parameters:
+      osName: windows
+      osVersion: RS5
+      kind: mlnet
+      architecture: x64
+      pool: Hosted VS2017
+      queue: Windows.10.Amd64.ClientRS5.Open
+      csproj: src\benchmarks\real-world\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj
+      runCategories: 'mldotnet'
+      libraryVersion: rolling
+      frameworks: # for rolling ML.NET version we want to use stable .NET Core release (to detect regressions in ML.NET)
+        - netcoreapp3.0
+
   # Windows x64 Roslyn benchmarks
   - template: /eng/performance/benchmark_jobs.yml
     parameters:
@@ -157,7 +173,7 @@ jobs:
         - netcoreapp2.2
         - netcoreapp2.1
 
-  # Ubuntu 1804 x64 ML.NET benchmarks
+  # Ubuntu 1804 x64 ML.NET benchmarks: fixed ML.NET, rolling .NET Core
   - template: /eng/performance/benchmark_jobs.yml
     parameters:
       osName: ubuntu
@@ -169,10 +185,27 @@ jobs:
       container: ubuntu_x64_build_container
       runCategories: 'mldotnet'
       csproj: src/benchmarks/real-world/Microsoft.ML.Benchmarks/Microsoft.ML.Benchmarks.csproj
-      frameworks: # for ML.NET jobs we want to check .NET Core 3.0 and 5.0 only
+      libraryVersion: fixed
+      frameworks: # for fixed ML.NET version we want to check latest .NET Core bits (to detect regressions in .NET Core)
         - netcoreapp5.0
         - netcoreapp3.1
 
+  # Ubuntu 1804 x64 ML.NET benchmarks: rolling ML.NET, fixed .NET Core
+  - template: /eng/performance/benchmark_jobs.yml
+    parameters:
+      osName: ubuntu
+      osVersion: 1804
+      kind: mlnet
+      architecture: x64
+      pool: Hosted Ubuntu 1604
+      queue: Ubuntu.1804.Amd64.Open
+      container: ubuntu_x64_build_container
+      runCategories: 'mldotnet'
+      csproj: src/benchmarks/real-world/Microsoft.ML.Benchmarks/Microsoft.ML.Benchmarks.csproj
+      libraryVersion: rolling
+      frameworks: # for rolling ML.NET version we want to use stable .NET Core release (to detect regressions in ML.NET)
+        - netcoreapp3.0
+
   # Ubuntu 1804 x64 Roslyn benchmarks
   - template: /eng/performance/benchmark_jobs.yml
     parameters:
@@ -237,7 +270,7 @@ jobs:
         - netcoreapp5.0
         - netcoreapp3.1
 
-  # Windows x64 ML.NET benchmarks
+  # Windows x64 ML.NET benchmarks: fixed ML.NET, rolling .NET Core
   - template: /eng/performance/benchmark_jobs.yml
     parameters:
       osName: windows
@@ -248,10 +281,26 @@ jobs:
       queue: Windows.10.Amd64.19H1.Tiger.Perf # using a dedicated private Helix queue (perftigers)
       csproj: src\benchmarks\real-world\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj
       runCategories: 'mldotnet'
-      frameworks: # for private jobs we want to benchmark .NET Core 3.0 and 5.0 only
+      libraryVersion: fixed
+      frameworks: # for fixed ML.NET version we want to check latest .NET Core bits (to detect regressions in .NET Core)
         - netcoreapp5.0
         - netcoreapp3.1
 
+  # Windows x64 ML.NET benchmarks: rolling ML.NET, fixed .NET Core
+  - template: /eng/performance/benchmark_jobs.yml
+    parameters:
+      osName: windows
+      osVersion: 19H1
+      kind: mlnet
+      architecture: x64
+      pool: Hosted VS2017
+      queue: Windows.10.Amd64.19H1.Tiger.Perf # using a dedicated private Helix queue (perftigers)
+      csproj: src\benchmarks\real-world\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj
+      runCategories: 'mldotnet'
+      libraryVersion: rolling
+      frameworks: # for rolling ML.NET version we want to use stable .NET Core release (to detect regressions in ML.NET)
+        - netcoreapp3.0
+
   # Windows x64 Roslyn benchmarks
   - template: /eng/performance/benchmark_jobs.yml
     parameters:
@@ -283,7 +332,7 @@ jobs:
         - netcoreapp5.0
         - netcoreapp3.1
 
-  # Ubuntu 1804 x64 ML.NET benchmarks
+  # Ubuntu 1804 x64 ML.NET benchmarks: fixed ML.NET, rolling .NET Core
   - template: /eng/performance/benchmark_jobs.yml
     parameters:
       osName: ubuntu
@@ -295,10 +344,27 @@ jobs:
       container: ubuntu_x64_build_container
       csproj: src/benchmarks/real-world/Microsoft.ML.Benchmarks/Microsoft.ML.Benchmarks.csproj
       runCategories: 'mldotnet'
-      frameworks: # for private jobs we want to benchmark .NET Core 3.0 and 5.0 only
+      libraryVersion: fixed
+      frameworks: # for fixed ML.NET version we want to check latest .NET Core bits (to detect regressions in .NET Core)
         - netcoreapp5.0
         - netcoreapp3.1
-
+
+  # Ubuntu 1804 x64 ML.NET benchmarks: rolling ML.NET, fixed .NET Core
+  - template: /eng/performance/benchmark_jobs.yml
+    parameters:
+      osName: ubuntu
+      osVersion: 1804
+      kind: mlnet
+      architecture: x64
+      pool: Hosted Ubuntu 1604
+      queue: Ubuntu.1804.Amd64.Tiger.Perf # using a dedicated private Helix queue (perftigers)
+      container: ubuntu_x64_build_container
+      csproj: src/benchmarks/real-world/Microsoft.ML.Benchmarks/Microsoft.ML.Benchmarks.csproj
+      runCategories: 'mldotnet'
+      libraryVersion: rolling
+      frameworks: # for rolling ML.NET version we want to use stable .NET Core release (to detect regressions in ML.NET)
+        - netcoreapp3.0
+
   # Ubuntu 1804 x64 Roslyn benchmarks
   - template: /eng/performance/benchmark_jobs.yml
     parameters:
@@ -351,7 +417,7 @@ jobs:
         - netcoreapp5.0
         - netcoreapp3.1
 
-  # Windows x64 ML.NET benchmarks
+  # Windows x64 ML.NET benchmarks: fixed ML.NET, rolling .NET Core
   - template: /eng/performance/benchmark_jobs.yml
     parameters:
       osName: windows
@@ -362,10 +428,26 @@ jobs:
       queue: Windows.10.Amd64.ClientRS1.Perf # using a dedicated private Helix queue (perfsnakes)
       csproj: src\benchmarks\real-world\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj
       runCategories: 'mldotnet'
-      frameworks: # for private jobs we want to benchmark .NET Core 5.0 and 3.1
+      libraryVersion: fixed
+      frameworks: # for fixed ML.NET version we want to check latest .NET Core bits (to detect regressions in .NET Core)
         - netcoreapp5.0
         - netcoreapp3.1
 
+  # Windows x64 ML.NET benchmarks: rolling ML.NET, fixed .NET Core
+  - template: /eng/performance/benchmark_jobs.yml
+    parameters:
+      osName: windows
+      osVersion: RS1
+      kind: mlnet
+      architecture: x64
+      pool: Hosted VS2017
+      queue: Windows.10.Amd64.ClientRS1.Perf # using a dedicated private Helix queue (perfsnakes)
+      csproj: src\benchmarks\real-world\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj
+      runCategories: 'mldotnet'
+      libraryVersion: rolling
+      frameworks: # for rolling ML.NET version we want to use stable .NET Core release (to detect regressions in ML.NET)
+        - netcoreapp3.0
+
   # Windows x64 Roslyn benchmarks
   - template: /eng/performance/benchmark_jobs.yml
     parameters:
@@ -397,7 +479,7 @@ jobs:
         - netcoreapp5.0
         - netcoreapp3.1
 
-  # Ubuntu 1604 x64 ML.NET benchmarks
+  # Ubuntu 1604 x64 ML.NET benchmarks: fixed ML.NET, rolling .NET Core
   - template: /eng/performance/benchmark_jobs.yml
     parameters:
       osName: ubuntu
@@ -409,10 +491,27 @@ jobs:
       container: ubuntu_x64_build_container
       csproj: src/benchmarks/real-world/Microsoft.ML.Benchmarks/Microsoft.ML.Benchmarks.csproj
       runCategories: 'mldotnet'
-      frameworks: # for private jobs we want to benchmark .NET Core 5.0 and 3.1
+      libraryVersion: fixed
+      frameworks: # for fixed ML.NET version we want to check latest .NET Core bits (to detect regressions in .NET Core)
         - netcoreapp5.0
         - netcoreapp3.1
-
+
+  # Ubuntu 1604 x64 ML.NET benchmarks: rolling ML.NET, fixed .NET Core
+  - template: /eng/performance/benchmark_jobs.yml
+    parameters:
+      osName: ubuntu
+      osVersion: 1604
+      kind: mlnet
+      architecture: x64
+      pool: Hosted Ubuntu 1604
+      queue: Ubuntu.1604.Amd64.Perf # using a dedicated private Helix queue (perfsnakes)
+      container: ubuntu_x64_build_container
+      csproj: src/benchmarks/real-world/Microsoft.ML.Benchmarks/Microsoft.ML.Benchmarks.csproj
+      runCategories: 'mldotnet'
+      libraryVersion: rolling
+      frameworks: # for rolling ML.NET version we want to use stable .NET Core release (to detect regressions in ML.NET)
+        - netcoreapp3.0
+
   # Ubuntu 1604 x64 Roslyn benchmarks
   - template: /eng/performance/benchmark_jobs.yml
     parameters:

eng/Versions.props

@@ -12,6 +12,7 @@
       https://dotnetfeed.blob.core.windows.net/aspnet-aspnetcore/index.json;
       https://dotnet.myget.org/F/symreader-converter/api/v3/index.json;
       https://dotnet.myget.org/F/symreader/api/v3/index.json;
+      https://pkgs.dev.azure.com/dnceng/public/_packaging/MachineLearning/nuget/v3/index.json;
     </RestoreSources>
   </PropertyGroup>
   <!--Package versions-->

eng/performance/benchmark_jobs.yml

@@ -8,7 +8,8 @@ parameters:
   container: ''         # optional -- id of the container
   csproj: ''            # required -- relative path to csproj with benchmarks
   frameworks: []        # required -- list of frameworks. Allowed values: net461, netcoreapp2.1, netcoreapp2.2, netcoreapp3.0, corert
-  runCategories: ''     # required -- string of space separated categories supplied to benchmark dotnet
+  runCategories: ''     # required -- string of space separated categories supplied to BenchmarkDotNet
+  libraryVersion: ''    # optional -- fixed|rolling, used by ML.NET
 
 jobs:
 - template: ../common/templates/jobs/jobs.yml
@@ -17,8 +18,8 @@ jobs:
     enablePublishBuildArtifacts: true
     helixRepo: dotnet/performance
     jobs:
-      - job: '${{ parameters.osName }}_${{ parameters.osVersion }}_${{ parameters.architecture }}_${{ parameters.kind }}'
-        displayName: '${{ parameters.osName }} ${{ parameters.osVersion }} ${{ parameters.architecture }} ${{ parameters.kind }}'
+      - job: '${{ parameters.osName }}_${{ parameters.osVersion }}_${{ parameters.architecture }}_${{ parameters.kind }}_${{ parameters.libraryVersion }}'
+        displayName: '${{ parameters.osName }} ${{ parameters.osVersion }} ${{ parameters.architecture }} ${{ parameters.kind }} ${{ parameters.libraryVersion }}'
         timeoutInMinutes: 320
         variables:
         - ${{ if eq(parameters.osName, 'windows') }}:
@@ -92,9 +93,9 @@ jobs:
             ${{ each framework in parameters.frameworks }}:
               ${{ framework }}:
                 _Framework: ${{ framework }}
-                _Configs: CompilationMode=Tiered RunKind="${{ parameters.kind }}"
+                _Configs: CompilationMode=Tiered RunKind="${{ parameters.kind }}${{ parameters.libraryVersion }}"
                 _PerfLabArguments: $(PerfLabArguments)
-                _BuildConfig: ${{ parameters.architecture }}_$(_Framework)_${{ parameters.kind }} # needs to be unique to avoid logs overwriting in mc.dot.net
+                _BuildConfig: ${{ parameters.architecture }}_$(_Framework)_${{ parameters.kind }}_${{ parameters.libraryVersion }} # needs to be unique to avoid logs overwriting in mc.dot.net
         steps:
         - checkout: self
           clean: true
@@ -112,6 +113,7 @@ jobs:
             Creator: $(Creator)
             Architecture: ${{ parameters.architecture }}
             TargetCsproj: ${{ parameters.csproj }}
+            LibraryVersion: ${{ parameters.libraryVersion }}
             WorkItemTimeout: 4:00 # 4 hours
             Python: $(Python)
             ${{ if eq(parameters.osName, 'windows') }}:

eng/performance/helix.proj

@@ -16,6 +16,10 @@
     <WorkItemCommand>$(WorkItemCommand) $(CliArguments)</WorkItemCommand>
   </PropertyGroup>
 
+  <PropertyGroup Condition="'$(LibraryVersion)' != ''">
+    <WorkItemCommand>$(WorkItemCommand) --lib-version $(LibraryVersion)</WorkItemCommand>
+  </PropertyGroup>
+
   <ItemGroup>
     <HelixCorrelationPayload Include="$(CorrelationPayloadDirectory)">
       <PayloadDirectory>%(Identity)</PayloadDirectory>

scripts/benchmarks_ci.py

@@ -122,6 +122,13 @@ def add_arguments(parser: ArgumentParser) -> ArgumentParser:
         type=str,
         help='Product repository.'
     )
+    parser.add_argument(
+        '--lib-version',
+        dest='lib_version',
+        required=False,
+        type=str,
+        help='Version of the tested library. Supported only by some projects'
+    )
 
     def __is_valid_datetime(dt: str) -> str:
         try:
@@ -213,6 +220,10 @@ def __main(args: list) -> int:
         target_framework_monikers
     )
 
+    # set the product version using Env Vars handled by the MSBuild project file
+    if args.lib_version:
+        os.environ['PERFLAB_LIBRARY_VERSION'] = args.lib_version
+
     # dotnet --info
     dotnet.info(verbose=verbose)
 

src/benchmarks/real-world/Microsoft.ML.Benchmarks/Microsoft.ML.Benchmarks.csproj

@@ -4,10 +4,13 @@
     <OutputType>Exe</OutputType>
     <TargetFrameworks>$(PERFLAB_TARGET_FRAMEWORKS)</TargetFrameworks>
     <TargetFrameworks Condition="'$(TargetFrameworks)' == ''">netcoreapp3.0;netcoreapp5.0</TargetFrameworks>
+    <LibraryVersion>1.4.0</LibraryVersion>
+    <!-- PERFLAB_LIBRARY_VERSION is defined in .yml file, passed by CI python script -->
+    <LibraryVersion Condition="'$(PERFLAB_LIBRARY_VERSION)' == 'rolling'">1.5.0-*</LibraryVersion>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.ML" Version="1.4.0-preview2" />
+    <PackageReference Include="Microsoft.ML" Version="$(LibraryVersion)" />
   </ItemGroup>
 
   <ItemGroup>