Enable wasm aot microbenchmark run (#1902)

* copy back the helix job folder for diagnosis

* copy back the helix job folder for diagnosis

* hack:setting WasmNativeWorkload=false for wasm run. should be removed before PR

* fixed merge confict

* fix copyjob

* move copy helix job-* folder code to benchmarks_ci.py

* update benchmarkdotnet version

* add more log info for diagnosis

* suppress an unnecessory exception

* fix copy back job-* folder

* fix copy job

* correct helixuploadroot

* add error handling

* fix copy path

* enabled binlog

* rethrow

* add back copy job folder for diagnosis

* dotnet workload install wasm-tools to work around missing workload error

* run dotnet workload install wasm-tools from dotnet-wasm directory to use the same nuget.config

* install wasm-tools workload from perf directory

* install wasm-tools workload with a specific nuget.config with wasm feed

* fix path

* workdir of install wasm-tools

* Get PR#1891 from main

* update to new version of benchmarkdotnet for wasm aot fixes

* add -d to print diagnostic info for wasm_tools workload installation failure

* Allow to build with local BenchmarkDotNet sources

Make it easier to use custom BenchmarkDotNet sources. Example use:

    dotnet build -bl -c Release -p:BenchmarkDotNetSources="\myPath\git\BenchmarkDotNet\" src\benchmarks\micro

* Set _BenchmarkDotNetSourcesN conditionaly

To avoid:

    src\harness\BenchmarkDotNet.Extensions\BenchmarkDotNet.Extensions.csproj(6,5): error MSB4184: The expression "[MSBuild]::NormalizeDirectory('')" cannot be evaluated. Parameter "path" cannot have zero length.

* Use _BenchmarkDotNetSourcesN

* Use BenchmarkDotNet sources in payload

* update to newer benchmarkdotnet version

* remove testing code

* update bdn version

* remove duplicated section

* print out args.configuration for diagnosis

* print args

* print args

* minor fix

* looks up wasm in ExtraBenchmarkDotNetArguments

* check wasmEngine for wasm runs

* minor fix

* minor fix

* remove work around code for install wasm_tools workload, WasmNativeWorkload property. mark perf_file.WriteAllBytesAsync not run under wasm since async is not supported under wasm

* add back Directory.Build.props to see if can work around an aot build error

* disable FlushAsync test from wasm run since async is not supported.

Co-authored-by: Radek Doulik <radekdoulik@gmail.com>

Author: Lxiamail

docs/Directory.Build.props

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project>
+  <PropertyGroup>
+	  <WasmNativeWorkload>false</WasmNativeWorkload>
+  </PropertyGroup>
+</Project>

scripts/benchmarks_ci.py

@@ -157,7 +157,6 @@ def __is_valid_datetime(dt: str) -> str:
         action='store_true',
         help='Attempts to run the benchmarks without building.',
     )
-
     return parser
 
 
@@ -171,7 +170,6 @@ def __process_arguments(args: list):
     add_arguments(parser)
     return parser.parse_args(args)
 
-
 def __main(args: list) -> int:
     validate_supported_runtime()
     args = __process_arguments(args)
@@ -270,6 +268,5 @@ def __main(args: list) -> int:
             upload.upload(globpath, upload_container, UPLOAD_QUEUE, UPLOAD_TOKEN_VAR, UPLOAD_STORAGE_URI)
         # TODO: Archive artifacts.
 
-
 if __name__ == "__main__":
     __main(sys.argv[1:])

scripts/micro_benchmarks.py

@@ -313,8 +313,7 @@ def run(
         verbose,
         *run_args
     )
-
-
+            
 def __log_script_header(message: str):
     getLogger().info('-' * len(message))
     getLogger().info(message)
@@ -361,7 +360,6 @@ def __main(args: list) -> int:
                 verbose,
                 args
             )
-
         return 0
     except CalledProcessError as ex:
         getLogger().error(

scripts/performance/common.py

@@ -80,6 +80,12 @@ def helixpayload():
     '''
     return environ.get('HELIX_CORRELATION_PAYLOAD')
 
+def helixuploadroot():
+    '''
+    Returns the helix upload root. Will be None outside of helix.
+    '''
+    return environ.get('HELIX_WORKITEM_UPLOAD_ROOT')
+
 def runninginlab():
     return environ.get('PERFLAB_INLAB') is not None
 

src/benchmarks/micro/libraries/System.IO.FileSystem/Perf.File.cs

@@ -81,6 +81,7 @@ public void SetupWriteAllBytes()
         public void WriteAllBytes(int size) => File.WriteAllBytes(_testFilePath, _userBuffers[size]);
 
 #if !NETFRAMEWORK
+        [BenchmarkCategory(Categories.NoWASM)]
         [Benchmark]
         [Arguments(HalfKibibyte)]
         [Arguments(FourKibibytes)]

src/benchmarks/micro/libraries/System.IO.FileSystem/Perf.FileStream.cs

@@ -322,6 +322,7 @@ public void Flush(long fileSize, FileOptions options)
             }
         }
 
+        [BenchmarkCategory(Categories.NoWASM)]
         [Benchmark]
         [Arguments(OneKibibyte, FileOptions.None)]
         [Arguments(OneKibibyte, FileOptions.Asynchronous)]

src/benchmarks/micro/wasm/nuget.config

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <packageSources>
+    <clear />
+    <add key="dotnet-public" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/index.json" />
+    <add key="dotnet6" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet6/nuget/v3/index.json" />
+    <add key="nuget.org" value="https://api.nuget.org/v3/index.json" protocolVersion="3" />    
+    <add key="xamarin" value="https://pkgs.dev.azure.com/azure-public/vside/_packaging/xamarin-impl/nuget/v3/index.json" />
+    <add key="dotnet-eng" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json" />
+    <add key="dotnet-tools" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json" />
+    <add key="benchmark-dotnet-prerelease" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/benchmark-dotnet-prerelease/nuget/v3/index.json" />
+  </packageSources>
+</configuration>

src/harness/BenchmarkDotNet.Extensions/BenchmarkDotNet.Extensions.csproj

@@ -7,7 +7,7 @@
   <PropertyGroup Condition="'$(BenchmarkDotNetSources)' != ''">
     <_BenchmarkDotNetSourcesN>$([MSBuild]::NormalizeDirectory('$(BenchmarkDotNetSources)'))</_BenchmarkDotNetSourcesN>
   </PropertyGroup>
-  <ItemGroup>
+  <ItemGroup Condition="'$(BenchmarkDotNetSources)' == ''">
     <PackageReference Include="BenchmarkDotNet" Version="0.13.0.1579" />
     <PackageReference Include="BenchmarkDotNet.Diagnostics.Windows" Version="0.13.0.1579" />
   </ItemGroup>

src/harness/BenchmarkDotNet.Extensions/RecommendedConfig.cs

@@ -37,7 +37,7 @@ public static IConfig Create(
                     .DontEnforcePowerPlan(); // make sure BDN does not try to enforce High Performance power plan on Windows
 
                 // See https://github.com/dotnet/roslyn/issues/42393
-                job = job.WithArguments(new Argument[] { new MsBuildArgument("/p:DebugType=portable") });
+                job = job.WithArguments(new Argument[] { new MsBuildArgument("/p:DebugType=portable"), new MsBuildArgument("-bl:benchmarkdotnet.binlog") });
             }
 
             var config = ManualConfig.CreateEmpty()