[perf][nativeaot] Add Native AOT performance measurements for iOS

azure-pipelines.yml

@@ -333,6 +333,21 @@ jobs:
         projectFile: maui_scenarios_ios.proj
         dotnetVersionsLinks:
           main: https://aka.ms/dotnet/sdk/maui/net8.0.json
+        runtimeFlavor: mono
+
+  # Maui iOS scenario benchmarks
+  - template: /eng/performance/build_machine_matrix.yml
+    parameters:
+      jobTemplate: /eng/performance/scenarios.yml
+      buildMachines:
+        - osx-x64-ios-arm64
+      isPublic: false
+      jobParameters:
+        kind: maui_scenarios_ios
+        projectFile: maui_scenarios_ios.proj
+        dotnetVersionsLinks:
+          main: https://aka.ms/dotnet/sdk/maui/net8.0.json
+        runtimeFlavor: coreclr
 
   ## Maui scenario benchmarks
   #- template: /eng/performance/build_machine_matrix.yml

eng/performance/maui_scenarios_ios.proj

@@ -10,6 +10,7 @@
     <PreparePayloadOutDirectoryName>scenarios_out</PreparePayloadOutDirectoryName>
     <PreparePayloadWorkItemBaseDirectory Condition="'$(TargetsWindows)' == 'true'">$(CorrelationPayloadDirectory)$(PreparePayloadOutDirectoryName)\</PreparePayloadWorkItemBaseDirectory>
     <PreparePayloadWorkItemBaseDirectory Condition="'$(TargetsWindows)' != 'true'">$(CorrelationPayloadDirectory)$(PreparePayloadOutDirectoryName)/</PreparePayloadWorkItemBaseDirectory>
+    <OptionalCommandProps Condition="'$(RuntimeFlavor)' == 'coreclr'">--nativeaot true</OptionalCommandProps>
   </PropertyGroup>
 
   <Target Name="RemoveDotnetFromCorrelationStaging" BeforeTargets="BeforeTest">
@@ -24,30 +25,36 @@
   </ItemDefinitionGroup>
 
   <ItemGroup>
-    <MAUIiOSScenario Include="Maui iOS Default">
+    <MAUIiOSScenario Include="Xamarin iOS Default $(RuntimeFlavor)">
+      <ScenarioDirectoryName>xamarinios</ScenarioDirectoryName>
+      <PayloadDirectory>$(ScenariosDir)%(ScenarioDirectoryName)</PayloadDirectory>
+      <IPAName>XamariniOSDefault</IPAName>
+      <PackageName>com.companyname.XamariniOSDefault</PackageName>
+    </MAUIiOSScenario>
+    <MAUIiOSScenario Include="Maui iOS Default $(RuntimeFlavor)">
       <ScenarioDirectoryName>mauiios</ScenarioDirectoryName>
       <PayloadDirectory>$(ScenariosDir)%(ScenarioDirectoryName)</PayloadDirectory>
       <IPAName>MauiiOSDefault</IPAName>
       <PackageName>net.dot.mauitesting</PackageName>
     </MAUIiOSScenario>
+    <MAUIiOSScenario Include="Maui Blazor iOS Default $(RuntimeFlavor)">
+      <ScenarioDirectoryName>mauiblazorios</ScenarioDirectoryName>
+      <PayloadDirectory>$(ScenariosDir)%(ScenarioDirectoryName)</PayloadDirectory>
+      <IPAName>MauiBlazoriOSDefault</IPAName>
+      <PackageName>net.dot.mauiblazortesting</PackageName>
+    </MAUIiOSScenario>
     <!-- MessagingCenter was removed: https://github.com/dotnet/maui/pull/12582, needs to be replaced likely with something like: https://github.com/dotnet/maui/commit/762e07e04d1fdf8fca3edd948bab294c8762cd2c -->
     <!-- <MAUIiOSScenario Include="Maui iOS Podcast">
       <ScenarioDirectoryName>mauiiospodcast</ScenarioDirectoryName>
       <PayloadDirectory>$(ScenariosDir)%(ScenarioDirectoryName)</PayloadDirectory>
       <IPAName>MauiiOSPodcast</IPAName>
       <PackageName>net.dot.net.dot.netconf2021.maui</PackageName>
     </MAUIiOSScenario> -->
-    <MAUIiOSScenario Include="Maui Blazor iOS Default">
-      <ScenarioDirectoryName>mauiblazorios</ScenarioDirectoryName>
-      <PayloadDirectory>$(ScenariosDir)%(ScenarioDirectoryName)</PayloadDirectory>
-      <IPAName>MauiBlazoriOSDefault</IPAName>
-      <PackageName>net.dot.mauiblazortesting</PackageName>
-    </MAUIiOSScenario>
   </ItemGroup>
 
   <ItemGroup>
     <PreparePayloadWorkItem Include="@(MAUIiOSScenario)">
-      <Command>$(Python) pre.py publish -f $(PERFLAB_Framework)-ios --self-contained -c Release -r ios-arm64 -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName); cd ../; zip -r %(PreparePayloadWorkItem.ScenarioDirectoryName).zip %(PreparePayloadWorkItem.ScenarioDirectoryName)</Command> 
+      <Command>$(Python) pre.py publish -f $(PERFLAB_Framework)-ios --self-contained -c Release -r ios-arm64 $(OptionalCommandProps) -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName); cd ../; zip -r %(PreparePayloadWorkItem.ScenarioDirectoryName).zip %(PreparePayloadWorkItem.ScenarioDirectoryName)</Command>
       <WorkingDirectory>%(PreparePayloadWorkItem.PayloadDirectory)</WorkingDirectory>
     </PreparePayloadWorkItem>
   </ItemGroup>
@@ -61,7 +68,35 @@
       <PreCommands>cp -r $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName) $HELIX_WORKITEM_ROOT/pub; mv $HELIX_WORKITEM_ROOT/pub/%(HelixWorkItem.IPAName).ipa $HELIX_WORKITEM_ROOT/pub/%(HelixWorkItem.IPAName).zip; unzip -d $HELIX_WORKITEM_ROOT/pub $HELIX_WORKITEM_ROOT/pub/%(HelixWorkItem.IPAName).zip; rm $HELIX_WORKITEM_ROOT/pub/%(HelixWorkItem.IPAName).zip</PreCommands>
       <Command>$(Python) test.py sod --scenario-name &quot;%(Identity)&quot;</Command>
     </HelixWorkItem>
-    <XHarnessAppBundleToTest Include="Device Startup - iOS Maui Default">
+    <XHarnessAppBundleToTest Include="Device Startup - iOS Xamarin Default $(RuntimeFlavor)">
+      <AppBundlePath>$(ScenariosDir)xamarinios.zip</AppBundlePath>
+      <WorkItemTimeout>00:15:00</WorkItemTimeout>
+      <TestTarget>ios-device</TestTarget>
+      <CustomCommands>
+        <![CDATA[
+            # PreCommands
+            export XHARNESSPATH=$XHARNESS_CLI_PATH
+
+            cp -v $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/xamarinios/XamariniOSDefault.ipa $HELIX_WORKITEM_ROOT/xamarinios/XamariniOSDefault.zip
+            mkdir $HELIX_WORKITEM_ROOT/xamarinios/pub
+            cp -v $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/xamarinios/versions.json $HELIX_WORKITEM_ROOT/xamarinios/pub/versions.json
+            unzip -d $HELIX_WORKITEM_ROOT/xamarinios $HELIX_WORKITEM_ROOT/xamarinios/XamariniOSDefault.zip
+            mv $HELIX_WORKITEM_ROOT/xamarinios/Payload/XamariniOSDefault.app $HELIX_WORKITEM_ROOT/xamarinios/XamariniOSDefault.app
+            cp -f embedded.mobileprovision $HELIX_WORKITEM_ROOT/xamarinios/XamariniOSDefault.app
+            cd $HELIX_WORKITEM_ROOT/xamarinios
+            sign XamariniOSDefault.app
+
+            # Testing commands
+            $(Python) test.py devicestartup --device-type ios --package-path XamariniOSDefault.app --package-name com.companyname.XamariniOSDefault --scenario-name "%(Identity)"
+            ((result=$?))
+
+            # Post commands
+            $(Python) post.py
+            exit $result
+          ]]>
+        </CustomCommands>
+    </XHarnessAppBundleToTest>
+    <XHarnessAppBundleToTest Include="Device Startup - iOS Maui Default $(RuntimeFlavor)">
       <AppBundlePath>$(ScenariosDir)mauiios.zip</AppBundlePath>
       <WorkItemTimeout>00:15:00</WorkItemTimeout>
       <TestTarget>ios-device</TestTarget>
@@ -89,7 +124,7 @@
           ]]>
         </CustomCommands>
     </XHarnessAppBundleToTest>
-    <XHarnessAppBundleToTest Include="Device Startup - iOS Maui Blazor Default">
+    <XHarnessAppBundleToTest Include="Device Startup - iOS Maui Blazor Default $(RuntimeFlavor)">
       <AppBundlePath>$(ScenariosDir)mauiblazorios.zip</AppBundlePath>
       <WorkItemTimeout>00:15:00</WorkItemTimeout>
       <TestTarget>ios-device</TestTarget>

eng/performance/scenarios.yml

@@ -13,6 +13,7 @@ parameters:
   affinity: '0'         # optional -- Affinity bitmask to a specific machine in the pool as a decimal number (e.g. 1, 2, etc) 
   runEnvVars: []        # optional -- Environment variables to set for the benchmark run in the form of a multi-line object like: "runEnvVars:\n - DOTNET_GCgen0size=1E00000 \n - DOTNET_GCHeapCount=4 \n - DOTNET_GCTotalPhysicalMemory=200000000"
   additionalJobIdentifier: '' # optional -- Additional identifier to append to the job name (no spaces)
+  runtimeFlavor: ''     # optional -- Runtime flavor used for scenarios
 
 jobs:
 - template: ../common/templates/jobs/jobs.yml
@@ -21,8 +22,8 @@ jobs:
     enablePublishBuildArtifacts: true
     helixRepo: dotnet/performance
     jobs:
-      - job: '${{ parameters.osName }}_${{ parameters.osVersion }}_${{ parameters.architecture }}_${{ parameters.kind }}_${{ parameters.machinePool }}${{ parameters.additionalJobIdentifier }}'
-        displayName: '${{ parameters.osName }} ${{ parameters.osVersion }} ${{ parameters.architecture }} ${{ parameters.kind }} ${{ parameters.machinePool }} ${{ parameters.additionalJobIdentifier }}'
+      - job: '${{ parameters.osName }}_${{ parameters.osVersion }}_${{ parameters.architecture }}_${{ parameters.kind }}_${{ parameters.runtimeFlavor }}_${{ parameters.machinePool }}${{ parameters.additionalJobIdentifier }}'
+        displayName: '${{ parameters.osName }} ${{ parameters.osVersion }} ${{ parameters.architecture }} ${{ parameters.kind }} ${{ parameters.runtimeFlavor }} ${{ parameters.machinePool }} ${{ parameters.additionalJobIdentifier }}'
         timeoutInMinutes: 320
         variables:
         - name: AffinityValue # Make the affinity value available to the csproj generation
@@ -158,7 +159,7 @@ jobs:
               $(Python) -m pip install urllib3==1.26.15
               $(Python) -m pip install requests
               .\src\scenarios\init.ps1 -DotnetDirectory $(CorrelationStaging)dotnet
-              dotnet msbuild .\eng\performance\${{ parameters.projectFile }} /restore /t:PreparePayloadWorkItems /bl:.\artifacts\log\$(_BuildConfig)\PrepareWorkItemPayloads.binlog
+              dotnet msbuild .\eng\performance\${{ parameters.projectFile }} /restore /t:PreparePayloadWorkItems /p:RuntimeFlavor=${{ parameters.runtimeFlavor }} /bl:.\artifacts\log\$(_BuildConfig)\PrepareWorkItemPayloads.binlog
             displayName: Prepare scenarios
             env:
               CorrelationPayloadDirectory: $(CorrelationStaging)
@@ -193,7 +194,7 @@ jobs:
               $(Python) -m pip install --user urllib3==1.26.15
               $(Python) -m pip install --user requests
               . ./src/scenarios/init.sh -dotnetdir $(CorrelationStaging)dotnet
-              dotnet msbuild ./eng/performance/${{ parameters.projectFile }} /restore /t:PreparePayloadWorkItems /bl:./artifacts/log/$(_BuildConfig)/PrepareWorkItemPayloads.binlog
+              dotnet msbuild ./eng/performance/${{ parameters.projectFile }} /restore /t:PreparePayloadWorkItems /p:RuntimeFlavor=${{ parameters.runtimeFlavor }} /bl:./artifacts/log/$(_BuildConfig)/PrepareWorkItemPayloads.binlog
             displayName: Prepare scenarios
             env:
               CorrelationPayloadDirectory: $(CorrelationStaging)
@@ -221,7 +222,7 @@ jobs:
               $(Python) -m pip install --user urllib3==1.26.15
               $(Python) -m pip install --user requests
               . ./src/scenarios/init.sh -dotnetdir $(CorrelationStaging)dotnet
-              dotnet msbuild ./eng/performance/${{ parameters.projectFile }} /restore /t:PreparePayloadWorkItems /bl:./artifacts/log/$(_BuildConfig)/PrepareWorkItemPayloads.binlog
+              dotnet msbuild ./eng/performance/${{ parameters.projectFile }} /restore /t:PreparePayloadWorkItems /p:RuntimeFlavor=${{ parameters.runtimeFlavor }} /bl:./artifacts/log/$(_BuildConfig)/PrepareWorkItemPayloads.binlog
             displayName: Prepare scenarios
             env:
               CorrelationPayloadDirectory: $(CorrelationStaging)
@@ -243,4 +244,5 @@ jobs:
             Creator: $(Creator)
             Architecture: ${{ parameters.architecture }}
             WorkItemTimeout: 4:00 # 4 hours
-            Python: $(Python)
+            Python: $(Python)
+            RuntimeFlavor: ${{ parameters.runtimeFlavor }}

eng/performance/send-to-helix.yml

@@ -23,9 +23,10 @@ parameters:
   condition: succeeded()                 # optional -- condition for step to execute; defaults to succeeded()
   continueOnError: false                 # optional -- determines whether to continue the build if the step errors; defaults to false
   ProjectFile: helix.proj                # optional -- override default project used to send benchmarks
+  RuntimeFlavor: ''                      # optional -- runtime flavor used for scenarios
 
 steps:
-  - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 -warnaserror 0 $env:BUILD_SOURCESDIRECTORY\eng\performance\${{ parameters.ProjectFile }} /restore /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"'
+  - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 -warnaserror 0 $env:BUILD_SOURCESDIRECTORY\eng\performance\${{ parameters.ProjectFile }} /restore /t:Test /p:RuntimeFlavor=${{ parameters.RuntimeFlavor }} /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"'
     displayName: ${{ parameters.DisplayNamePrefix }} (Windows)
     env:
       Python: ${{ parameters.Python }}
@@ -51,7 +52,7 @@ steps:
       SYSTEM_ACCESSTOKEN: $(System.AccessToken)
     condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT'))
     continueOnError: ${{ parameters.continueOnError }}
-  - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh --warnaserror false $BUILD_SOURCESDIRECTORY/eng/performance/${{ parameters.ProjectFile }} /restore /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog
+  - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh --warnaserror false $BUILD_SOURCESDIRECTORY/eng/performance/${{ parameters.ProjectFile }} /restore /t:Test /p:RuntimeFlavor=${{ parameters.RuntimeFlavor }} /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog
     displayName: ${{ parameters.DisplayNamePrefix }} (Unix)
     env:
       Python: ${{ parameters.Python }}

src/scenarios/mauiblazorios/pre.py

@@ -72,8 +72,8 @@
 remove_aab_files(output_dir)
 
 # Copy the MauiVersion to a file so we have it on the machine
-maui_version = get_version_from_dll_powershell_ios(rf"./{const.APPDIR}/obj/Release/{precommands.framework}/ios-arm64/ipa/Payload/{EXENAME}.app/Microsoft.Maui.dll")
+maui_version = get_version_from_dll_powershell_ios(rf"./{const.APPDIR}/obj/Release/net8.0-ios/ios-arm64/linked/Microsoft.Maui.dll")
 version_dict = { "mauiVersion": maui_version }
 versions_write_json(version_dict, rf"{output_dir}/versions.json")
-print(f"Versions: {version_dict} from location " + rf"./{const.APPDIR}/obj/Release/{precommands.framework}/ios-arm64/ipa/Payload/{EXENAME}.app/Microsoft.Maui.dll")
+print(f"Versions: {version_dict} from location " + rf"./{const.APPDIR}/obj/Release/net8.0-ios/ios-arm64/linked/Microsoft.Maui.dll")
 

src/scenarios/mauiios/pre.py

@@ -35,7 +35,7 @@
 remove_aab_files(output_dir)
 
 # Copy the MauiVersion to a file so we have it on the machine
-maui_version = get_version_from_dll_powershell_ios(rf"./{const.APPDIR}/obj/Release/{precommands.framework}/ios-arm64/ipa/Payload/{EXENAME}.app/Microsoft.Maui.dll")
+maui_version = get_version_from_dll_powershell_ios(rf"./{const.APPDIR}/obj/Release/net8.0-ios/ios-arm64/linked/Microsoft.Maui.dll")
 version_dict = { "mauiVersion": maui_version }
 versions_write_json(version_dict, rf"{output_dir}/versions.json")
-print(f"Versions: {version_dict} from location " + rf"./{const.APPDIR}/obj/Release/{precommands.framework}/ios-arm64/ipa/Payload/{EXENAME}.app/Microsoft.Maui.dll")
+print(f"Versions: {version_dict} from location " + rf"./{const.APPDIR}/obj/Release/net8.0-ios/ios-arm64/linked/Microsoft.Maui.dll")

src/scenarios/shared/precommands.py

@@ -91,6 +91,7 @@ def __init__(self):
         self.operation = args.operation
         self.framework = args.framework
         self.runtime_identifier = args.runtime
+        self.nativeaot = args.nativeaot
         self.msbuild = args.msbuild
         print(self.msbuild)
         self.msbuildstatic = args.msbuildstatic
@@ -149,6 +150,10 @@ def add_common_arguments(self, parser: ArgumentParser):
                             dest='runtime',
                             metavar='runtime',
                             help='runtime for build or publish - ex: win-x64')
+        parser.add_argument('-n', '--nativeaot',
+                            dest='nativeaot',
+                            metavar='nativeaot',
+                            help='use Native AOT runtime for build or publish')
         parser.add_argument('--msbuild',
                             dest='msbuild',
                             metavar='msbuild',
@@ -202,6 +207,9 @@ def execute(self, build_args: list = []):
                 build_args.append('--self-contained')
             elif self.no_self_contained:
                 build_args.append('--no-self-contained')
+            if self.nativeaot:
+                build_args.append('/p:PublishAot=true')
+                build_args.append('/p:PublishAotUsingRuntimePack=true')
             self._publish(configuration=self.configuration, runtime_identifier=self.runtime_identifier, framework=self.framework, output=self.output, build_args=build_args)
         if self.operation == CROSSGEN:
             startup_args = [

src/scenarios/xamarinios/post.py

@@ -0,0 +1,7 @@
+'''
+post cleanup script
+'''
+
+from shared.postcommands import clean_directories
+
+clean_directories()

src/scenarios/xamarinios/pre.py

@@ -0,0 +1,41 @@
+'''
+pre-command
+'''
+import shutil
+import sys
+import subprocess
+from performance.logger import setup_loggers, getLogger
+from shared import const
+from shared.mauisharedpython import remove_aab_files, install_versioned_maui
+from shared.precommands import PreCommands
+from shared.versionmanager import versions_write_json, get_version_from_dll_powershell_ios
+from test import EXENAME
+
+setup_loggers(True)
+
+precommands = PreCommands()
+install_versioned_maui(precommands)
+
+# Setup the Xamarin folder
+precommands.new(template='ios',
+                output_dir=const.APPDIR,
+                bin_dir=const.BINDIR,
+                exename=EXENAME,
+                working_directory=sys.path[0],
+                no_restore=False)
+
+# Build the APK
+shutil.copy('./MauiNuGet.config', './app/Nuget.config')
+precommands.execute(['/p:_RequireCodeSigning=false', '/p:ApplicationId=net.dot.xamarintesting'])
+
+# Remove the aab files as we don't need them, this saves space
+output_dir = const.PUBDIR
+if precommands.output:
+    output_dir = precommands.output
+remove_aab_files(output_dir)
+
+# Copy the XamarinVersion to a file so we have it on the machine
+xamarin_version = get_version_from_dll_powershell_ios(rf"./{const.APPDIR}/obj/Release/net8.0-ios/ios-arm64/linked/Microsoft.iOS.dll")
+version_dict = { "xamarinVersion": xamarin_version }
+versions_write_json(version_dict, rf"{output_dir}/versions.json")
+print(f"Versions: {version_dict} from location " + rf"./{const.APPDIR}/obj/Release/net8.0-ios/ios-arm64/linked/Microsoft.iOS.dll")

src/scenarios/xamarinios/test.py

@@ -0,0 +1,16 @@
+'''
+Mobile Xamarin App
+'''
+from shared.const import PUBDIR
+from shared.runner import TestTraits, Runner
+from shared.versionmanager import versions_read_json_file_save_env
+
+EXENAME = 'XamariniOSDefault'
+
+if __name__ == "__main__":    
+    versions_read_json_file_save_env(rf"./{PUBDIR}/versions.json")
+
+    traits = TestTraits(exename=EXENAME, 
+                        guiapp='false',
+                        )
+    Runner(traits).run()