Skip to content

Commit

Permalink
[perf][nativeaot] Add Native AOT performance measurements for iOS (#3145
Browse files Browse the repository at this point in the history
)
  • Loading branch information
kotlarmilos authored Jul 21, 2023
1 parent a3cea63 commit e5ca99f
Show file tree
Hide file tree
Showing 10 changed files with 147 additions and 22 deletions.
15 changes: 15 additions & 0 deletions azure-pipelines.yml
Original file line number Diff line number Diff line change
Expand Up @@ -333,6 +333,21 @@ jobs:
projectFile: maui_scenarios_ios.proj
dotnetVersionsLinks:
main: https://aka.ms/dotnet/sdk/maui/net8.0.json
runtimeFlavor: mono

# Maui iOS scenario benchmarks
- template: /eng/performance/build_machine_matrix.yml
parameters:
jobTemplate: /eng/performance/scenarios.yml
buildMachines:
- osx-x64-ios-arm64
isPublic: false
jobParameters:
kind: maui_scenarios_ios
projectFile: maui_scenarios_ios.proj
dotnetVersionsLinks:
main: https://aka.ms/dotnet/sdk/maui/net8.0.json
runtimeFlavor: coreclr

## Maui scenario benchmarks
#- template: /eng/performance/build_machine_matrix.yml
Expand Down
55 changes: 45 additions & 10 deletions eng/performance/maui_scenarios_ios.proj
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
<PreparePayloadOutDirectoryName>scenarios_out</PreparePayloadOutDirectoryName>
<PreparePayloadWorkItemBaseDirectory Condition="'$(TargetsWindows)' == 'true'">$(CorrelationPayloadDirectory)$(PreparePayloadOutDirectoryName)\</PreparePayloadWorkItemBaseDirectory>
<PreparePayloadWorkItemBaseDirectory Condition="'$(TargetsWindows)' != 'true'">$(CorrelationPayloadDirectory)$(PreparePayloadOutDirectoryName)/</PreparePayloadWorkItemBaseDirectory>
<OptionalCommandProps Condition="'$(RuntimeFlavor)' == 'coreclr'">--nativeaot true</OptionalCommandProps>
</PropertyGroup>

<Target Name="RemoveDotnetFromCorrelationStaging" BeforeTargets="BeforeTest">
Expand All @@ -24,30 +25,36 @@
</ItemDefinitionGroup>

<ItemGroup>
<MAUIiOSScenario Include="Maui iOS Default">
<MAUIiOSScenario Include="Xamarin iOS Default $(RuntimeFlavor)">
<ScenarioDirectoryName>xamarinios</ScenarioDirectoryName>
<PayloadDirectory>$(ScenariosDir)%(ScenarioDirectoryName)</PayloadDirectory>
<IPAName>XamariniOSDefault</IPAName>
<PackageName>com.companyname.XamariniOSDefault</PackageName>
</MAUIiOSScenario>
<MAUIiOSScenario Include="Maui iOS Default $(RuntimeFlavor)">
<ScenarioDirectoryName>mauiios</ScenarioDirectoryName>
<PayloadDirectory>$(ScenariosDir)%(ScenarioDirectoryName)</PayloadDirectory>
<IPAName>MauiiOSDefault</IPAName>
<PackageName>net.dot.mauitesting</PackageName>
</MAUIiOSScenario>
<MAUIiOSScenario Include="Maui Blazor iOS Default $(RuntimeFlavor)">
<ScenarioDirectoryName>mauiblazorios</ScenarioDirectoryName>
<PayloadDirectory>$(ScenariosDir)%(ScenarioDirectoryName)</PayloadDirectory>
<IPAName>MauiBlazoriOSDefault</IPAName>
<PackageName>net.dot.mauiblazortesting</PackageName>
</MAUIiOSScenario>
<!-- MessagingCenter was removed: https://github.com/dotnet/maui/pull/12582, needs to be replaced likely with something like: https://github.com/dotnet/maui/commit/762e07e04d1fdf8fca3edd948bab294c8762cd2c -->
<!-- <MAUIiOSScenario Include="Maui iOS Podcast">
<ScenarioDirectoryName>mauiiospodcast</ScenarioDirectoryName>
<PayloadDirectory>$(ScenariosDir)%(ScenarioDirectoryName)</PayloadDirectory>
<IPAName>MauiiOSPodcast</IPAName>
<PackageName>net.dot.net.dot.netconf2021.maui</PackageName>
</MAUIiOSScenario> -->
<MAUIiOSScenario Include="Maui Blazor iOS Default">
<ScenarioDirectoryName>mauiblazorios</ScenarioDirectoryName>
<PayloadDirectory>$(ScenariosDir)%(ScenarioDirectoryName)</PayloadDirectory>
<IPAName>MauiBlazoriOSDefault</IPAName>
<PackageName>net.dot.mauiblazortesting</PackageName>
</MAUIiOSScenario>
</ItemGroup>

<ItemGroup>
<PreparePayloadWorkItem Include="@(MAUIiOSScenario)">
<Command>$(Python) pre.py publish -f $(PERFLAB_Framework)-ios --self-contained -c Release -r ios-arm64 -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName); cd ../; zip -r %(PreparePayloadWorkItem.ScenarioDirectoryName).zip %(PreparePayloadWorkItem.ScenarioDirectoryName)</Command>
<Command>$(Python) pre.py publish -f $(PERFLAB_Framework)-ios --self-contained -c Release -r ios-arm64 $(OptionalCommandProps) -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName); cd ../; zip -r %(PreparePayloadWorkItem.ScenarioDirectoryName).zip %(PreparePayloadWorkItem.ScenarioDirectoryName)</Command>
<WorkingDirectory>%(PreparePayloadWorkItem.PayloadDirectory)</WorkingDirectory>
</PreparePayloadWorkItem>
</ItemGroup>
Expand All @@ -61,7 +68,35 @@
<PreCommands>cp -r $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName) $HELIX_WORKITEM_ROOT/pub; mv $HELIX_WORKITEM_ROOT/pub/%(HelixWorkItem.IPAName).ipa $HELIX_WORKITEM_ROOT/pub/%(HelixWorkItem.IPAName).zip; unzip -d $HELIX_WORKITEM_ROOT/pub $HELIX_WORKITEM_ROOT/pub/%(HelixWorkItem.IPAName).zip; rm $HELIX_WORKITEM_ROOT/pub/%(HelixWorkItem.IPAName).zip</PreCommands>
<Command>$(Python) test.py sod --scenario-name &quot;%(Identity)&quot;</Command>
</HelixWorkItem>
<XHarnessAppBundleToTest Include="Device Startup - iOS Maui Default">
<XHarnessAppBundleToTest Include="Device Startup - iOS Xamarin Default $(RuntimeFlavor)">
<AppBundlePath>$(ScenariosDir)xamarinios.zip</AppBundlePath>
<WorkItemTimeout>00:15:00</WorkItemTimeout>
<TestTarget>ios-device</TestTarget>
<CustomCommands>
<![CDATA[
# PreCommands
export XHARNESSPATH=$XHARNESS_CLI_PATH
cp -v $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/xamarinios/XamariniOSDefault.ipa $HELIX_WORKITEM_ROOT/xamarinios/XamariniOSDefault.zip
mkdir $HELIX_WORKITEM_ROOT/xamarinios/pub
cp -v $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/xamarinios/versions.json $HELIX_WORKITEM_ROOT/xamarinios/pub/versions.json
unzip -d $HELIX_WORKITEM_ROOT/xamarinios $HELIX_WORKITEM_ROOT/xamarinios/XamariniOSDefault.zip
mv $HELIX_WORKITEM_ROOT/xamarinios/Payload/XamariniOSDefault.app $HELIX_WORKITEM_ROOT/xamarinios/XamariniOSDefault.app
cp -f embedded.mobileprovision $HELIX_WORKITEM_ROOT/xamarinios/XamariniOSDefault.app
cd $HELIX_WORKITEM_ROOT/xamarinios
sign XamariniOSDefault.app
# Testing commands
$(Python) test.py devicestartup --device-type ios --package-path XamariniOSDefault.app --package-name com.companyname.XamariniOSDefault --scenario-name "%(Identity)"
((result=$?))
# Post commands
$(Python) post.py
exit $result
]]>
</CustomCommands>
</XHarnessAppBundleToTest>
<XHarnessAppBundleToTest Include="Device Startup - iOS Maui Default $(RuntimeFlavor)">
<AppBundlePath>$(ScenariosDir)mauiios.zip</AppBundlePath>
<WorkItemTimeout>00:15:00</WorkItemTimeout>
<TestTarget>ios-device</TestTarget>
Expand Down Expand Up @@ -89,7 +124,7 @@
]]>
</CustomCommands>
</XHarnessAppBundleToTest>
<XHarnessAppBundleToTest Include="Device Startup - iOS Maui Blazor Default">
<XHarnessAppBundleToTest Include="Device Startup - iOS Maui Blazor Default $(RuntimeFlavor)">
<AppBundlePath>$(ScenariosDir)mauiblazorios.zip</AppBundlePath>
<WorkItemTimeout>00:15:00</WorkItemTimeout>
<TestTarget>ios-device</TestTarget>
Expand Down
14 changes: 8 additions & 6 deletions eng/performance/scenarios.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ parameters:
affinity: '0' # optional -- Affinity bitmask to a specific machine in the pool as a decimal number (e.g. 1, 2, etc)
runEnvVars: [] # optional -- Environment variables to set for the benchmark run in the form of a multi-line object like: "runEnvVars:\n - DOTNET_GCgen0size=1E00000 \n - DOTNET_GCHeapCount=4 \n - DOTNET_GCTotalPhysicalMemory=200000000"
additionalJobIdentifier: '' # optional -- Additional identifier to append to the job name (no spaces)
runtimeFlavor: '' # optional -- Runtime flavor used for scenarios

jobs:
- template: ../common/templates/jobs/jobs.yml
Expand All @@ -21,8 +22,8 @@ jobs:
enablePublishBuildArtifacts: true
helixRepo: dotnet/performance
jobs:
- job: '${{ parameters.osName }}_${{ parameters.osVersion }}_${{ parameters.architecture }}_${{ parameters.kind }}_${{ parameters.machinePool }}${{ parameters.additionalJobIdentifier }}'
displayName: '${{ parameters.osName }} ${{ parameters.osVersion }} ${{ parameters.architecture }} ${{ parameters.kind }} ${{ parameters.machinePool }} ${{ parameters.additionalJobIdentifier }}'
- job: '${{ parameters.osName }}_${{ parameters.osVersion }}_${{ parameters.architecture }}_${{ parameters.kind }}_${{ parameters.runtimeFlavor }}_${{ parameters.machinePool }}${{ parameters.additionalJobIdentifier }}'
displayName: '${{ parameters.osName }} ${{ parameters.osVersion }} ${{ parameters.architecture }} ${{ parameters.kind }} ${{ parameters.runtimeFlavor }} ${{ parameters.machinePool }} ${{ parameters.additionalJobIdentifier }}'
timeoutInMinutes: 320
variables:
- name: AffinityValue # Make the affinity value available to the csproj generation
Expand Down Expand Up @@ -158,7 +159,7 @@ jobs:
$(Python) -m pip install urllib3==1.26.15
$(Python) -m pip install requests
.\src\scenarios\init.ps1 -DotnetDirectory $(CorrelationStaging)dotnet
dotnet msbuild .\eng\performance\${{ parameters.projectFile }} /restore /t:PreparePayloadWorkItems /bl:.\artifacts\log\$(_BuildConfig)\PrepareWorkItemPayloads.binlog
dotnet msbuild .\eng\performance\${{ parameters.projectFile }} /restore /t:PreparePayloadWorkItems /p:RuntimeFlavor=${{ parameters.runtimeFlavor }} /bl:.\artifacts\log\$(_BuildConfig)\PrepareWorkItemPayloads.binlog
displayName: Prepare scenarios
env:
CorrelationPayloadDirectory: $(CorrelationStaging)
Expand Down Expand Up @@ -193,7 +194,7 @@ jobs:
$(Python) -m pip install --user urllib3==1.26.15
$(Python) -m pip install --user requests
. ./src/scenarios/init.sh -dotnetdir $(CorrelationStaging)dotnet
dotnet msbuild ./eng/performance/${{ parameters.projectFile }} /restore /t:PreparePayloadWorkItems /bl:./artifacts/log/$(_BuildConfig)/PrepareWorkItemPayloads.binlog
dotnet msbuild ./eng/performance/${{ parameters.projectFile }} /restore /t:PreparePayloadWorkItems /p:RuntimeFlavor=${{ parameters.runtimeFlavor }} /bl:./artifacts/log/$(_BuildConfig)/PrepareWorkItemPayloads.binlog
displayName: Prepare scenarios
env:
CorrelationPayloadDirectory: $(CorrelationStaging)
Expand Down Expand Up @@ -221,7 +222,7 @@ jobs:
$(Python) -m pip install --user urllib3==1.26.15
$(Python) -m pip install --user requests
. ./src/scenarios/init.sh -dotnetdir $(CorrelationStaging)dotnet
dotnet msbuild ./eng/performance/${{ parameters.projectFile }} /restore /t:PreparePayloadWorkItems /bl:./artifacts/log/$(_BuildConfig)/PrepareWorkItemPayloads.binlog
dotnet msbuild ./eng/performance/${{ parameters.projectFile }} /restore /t:PreparePayloadWorkItems /p:RuntimeFlavor=${{ parameters.runtimeFlavor }} /bl:./artifacts/log/$(_BuildConfig)/PrepareWorkItemPayloads.binlog
displayName: Prepare scenarios
env:
CorrelationPayloadDirectory: $(CorrelationStaging)
Expand All @@ -243,4 +244,5 @@ jobs:
Creator: $(Creator)
Architecture: ${{ parameters.architecture }}
WorkItemTimeout: 4:00 # 4 hours
Python: $(Python)
Python: $(Python)
RuntimeFlavor: ${{ parameters.runtimeFlavor }}
5 changes: 3 additions & 2 deletions eng/performance/send-to-helix.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,10 @@ parameters:
condition: succeeded() # optional -- condition for step to execute; defaults to succeeded()
continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false
ProjectFile: helix.proj # optional -- override default project used to send benchmarks
RuntimeFlavor: '' # optional -- runtime flavor used for scenarios

steps:
- powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 -warnaserror 0 $env:BUILD_SOURCESDIRECTORY\eng\performance\${{ parameters.ProjectFile }} /restore /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"'
- powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 -warnaserror 0 $env:BUILD_SOURCESDIRECTORY\eng\performance\${{ parameters.ProjectFile }} /restore /t:Test /p:RuntimeFlavor=${{ parameters.RuntimeFlavor }} /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"'
displayName: ${{ parameters.DisplayNamePrefix }} (Windows)
env:
Python: ${{ parameters.Python }}
Expand All @@ -51,7 +52,7 @@ steps:
SYSTEM_ACCESSTOKEN: $(System.AccessToken)
condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT'))
continueOnError: ${{ parameters.continueOnError }}
- script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh --warnaserror false $BUILD_SOURCESDIRECTORY/eng/performance/${{ parameters.ProjectFile }} /restore /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog
- script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh --warnaserror false $BUILD_SOURCESDIRECTORY/eng/performance/${{ parameters.ProjectFile }} /restore /t:Test /p:RuntimeFlavor=${{ parameters.RuntimeFlavor }} /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog
displayName: ${{ parameters.DisplayNamePrefix }} (Unix)
env:
Python: ${{ parameters.Python }}
Expand Down
4 changes: 2 additions & 2 deletions src/scenarios/mauiblazorios/pre.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,8 +72,8 @@
remove_aab_files(output_dir)

# Copy the MauiVersion to a file so we have it on the machine
maui_version = get_version_from_dll_powershell_ios(rf"./{const.APPDIR}/obj/Release/{precommands.framework}/ios-arm64/ipa/Payload/{EXENAME}.app/Microsoft.Maui.dll")
maui_version = get_version_from_dll_powershell_ios(rf"./{const.APPDIR}/obj/Release/net8.0-ios/ios-arm64/linked/Microsoft.Maui.dll")
version_dict = { "mauiVersion": maui_version }
versions_write_json(version_dict, rf"{output_dir}/versions.json")
print(f"Versions: {version_dict} from location " + rf"./{const.APPDIR}/obj/Release/{precommands.framework}/ios-arm64/ipa/Payload/{EXENAME}.app/Microsoft.Maui.dll")
print(f"Versions: {version_dict} from location " + rf"./{const.APPDIR}/obj/Release/net8.0-ios/ios-arm64/linked/Microsoft.Maui.dll")

4 changes: 2 additions & 2 deletions src/scenarios/mauiios/pre.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
remove_aab_files(output_dir)

# Copy the MauiVersion to a file so we have it on the machine
maui_version = get_version_from_dll_powershell_ios(rf"./{const.APPDIR}/obj/Release/{precommands.framework}/ios-arm64/ipa/Payload/{EXENAME}.app/Microsoft.Maui.dll")
maui_version = get_version_from_dll_powershell_ios(rf"./{const.APPDIR}/obj/Release/net8.0-ios/ios-arm64/linked/Microsoft.Maui.dll")
version_dict = { "mauiVersion": maui_version }
versions_write_json(version_dict, rf"{output_dir}/versions.json")
print(f"Versions: {version_dict} from location " + rf"./{const.APPDIR}/obj/Release/{precommands.framework}/ios-arm64/ipa/Payload/{EXENAME}.app/Microsoft.Maui.dll")
print(f"Versions: {version_dict} from location " + rf"./{const.APPDIR}/obj/Release/net8.0-ios/ios-arm64/linked/Microsoft.Maui.dll")
8 changes: 8 additions & 0 deletions src/scenarios/shared/precommands.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ def __init__(self):
self.operation = args.operation
self.framework = args.framework
self.runtime_identifier = args.runtime
self.nativeaot = args.nativeaot
self.msbuild = args.msbuild
print(self.msbuild)
self.msbuildstatic = args.msbuildstatic
Expand Down Expand Up @@ -149,6 +150,10 @@ def add_common_arguments(self, parser: ArgumentParser):
dest='runtime',
metavar='runtime',
help='runtime for build or publish - ex: win-x64')
parser.add_argument('-n', '--nativeaot',
dest='nativeaot',
metavar='nativeaot',
help='use Native AOT runtime for build or publish')
parser.add_argument('--msbuild',
dest='msbuild',
metavar='msbuild',
Expand Down Expand Up @@ -202,6 +207,9 @@ def execute(self, build_args: list = []):
build_args.append('--self-contained')
elif self.no_self_contained:
build_args.append('--no-self-contained')
if self.nativeaot:
build_args.append('/p:PublishAot=true')
build_args.append('/p:PublishAotUsingRuntimePack=true')
self._publish(configuration=self.configuration, runtime_identifier=self.runtime_identifier, framework=self.framework, output=self.output, build_args=build_args)
if self.operation == CROSSGEN:
startup_args = [
Expand Down
7 changes: 7 additions & 0 deletions src/scenarios/xamarinios/post.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
'''
post cleanup script
'''

from shared.postcommands import clean_directories

clean_directories()
41 changes: 41 additions & 0 deletions src/scenarios/xamarinios/pre.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
'''
pre-command
'''
import shutil
import sys
import subprocess
from performance.logger import setup_loggers, getLogger
from shared import const
from shared.mauisharedpython import remove_aab_files, install_versioned_maui
from shared.precommands import PreCommands
from shared.versionmanager import versions_write_json, get_version_from_dll_powershell_ios
from test import EXENAME

setup_loggers(True)

precommands = PreCommands()
install_versioned_maui(precommands)

# Setup the Xamarin folder
precommands.new(template='ios',
output_dir=const.APPDIR,
bin_dir=const.BINDIR,
exename=EXENAME,
working_directory=sys.path[0],
no_restore=False)

# Build the APK
shutil.copy('./MauiNuGet.config', './app/Nuget.config')
precommands.execute(['/p:_RequireCodeSigning=false', '/p:ApplicationId=net.dot.xamarintesting'])

# Remove the aab files as we don't need them, this saves space
output_dir = const.PUBDIR
if precommands.output:
output_dir = precommands.output
remove_aab_files(output_dir)

# Copy the XamarinVersion to a file so we have it on the machine
xamarin_version = get_version_from_dll_powershell_ios(rf"./{const.APPDIR}/obj/Release/net8.0-ios/ios-arm64/linked/Microsoft.iOS.dll")
version_dict = { "xamarinVersion": xamarin_version }
versions_write_json(version_dict, rf"{output_dir}/versions.json")
print(f"Versions: {version_dict} from location " + rf"./{const.APPDIR}/obj/Release/net8.0-ios/ios-arm64/linked/Microsoft.iOS.dll")
16 changes: 16 additions & 0 deletions src/scenarios/xamarinios/test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
'''
Mobile Xamarin App
'''
from shared.const import PUBDIR
from shared.runner import TestTraits, Runner
from shared.versionmanager import versions_read_json_file_save_env

EXENAME = 'XamariniOSDefault'

if __name__ == "__main__":
versions_read_json_file_save_env(rf"./{PUBDIR}/versions.json")

traits = TestTraits(exename=EXENAME,
guiapp='false',
)
Runner(traits).run()

0 comments on commit e5ca99f

Please sign in to comment.