SPMI: Run 64-bit diffs on OSX.14.Arm64.Open

eng/pipelines/coreclr/superpmi-diffs.yml

@@ -53,11 +53,17 @@ extends:
           jobTemplate: /eng/pipelines/common/global-build-job.yml
           buildConfig: checked
           platforms:
-          - windows_x64
+          - osx_arm64
           - windows_x86
           jobParameters:
-            buildArgs: -s clr.alljits+clr.alljitscommunity+clr.spmi -c $(_BuildConfig)
+            buildArgs: -s clr.alljits+clr.spmi -c $(_BuildConfig)
             postBuildSteps:
+              # Build CLR assets for x64 as well as the target as we need an x64 mcs
+              - template: /eng/pipelines/common/templates/global-build-step.yml
+                parameters:
+                  buildArgs: -s clr.spmi -c $(_BuildConfig)
+                  archParameter: -arch x64
+                  displayName: Build SuperPMI
               - template: /eng/pipelines/common/upload-artifact-step.yml
                 parameters:
                   rootFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr
@@ -96,9 +102,9 @@ extends:
           jobTemplate: /eng/pipelines/coreclr/templates/superpmi-diffs-job.yml
           buildConfig: checked
           platforms:
-          - windows_x64
+          - osx_arm64
           - windows_x86
-          helixQueueGroup: ci
+          helixQueueGroup: superpmi-diffs
           helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
           jobParameters:
             condition: not(eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_jiteeversionguid.containsChange'], true))
@@ -114,7 +120,7 @@ extends:
           - windows_x64
           - windows_x86
           - linux_x64
-          helixQueueGroup: ci
+          helixQueueGroup: superpmi-diffs
           helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
           jobParameters:
             condition: not(eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_jiteeversionguid.containsChange'], true))

eng/pipelines/coreclr/templates/helix-queues-setup.yml

@@ -15,7 +15,7 @@ parameters:
 # 'ci' -- continuous integration ("merge")
 # 'libraries' -- libraries tests
 # 'cet' -- machines supporting CET technology
-# 'superpmi' -- for TeamProject 'internal', a smaller set of queues (one per architecture, not several) for SuperPMI collection
+# 'superpmi-diffs' -- as high capacity queues as possible since no OS testing is necessary for this
 # Many of these values are unused below, as we use the same queues for many of the scenarios.
 # However, having these options allows us to easily tune the queues we use for each scenario as needed based on new hardware support.
 
@@ -104,8 +104,11 @@ jobs:
 
     # OSX arm64
     - ${{ if eq(parameters.platform, 'osx_arm64') }}:
-      - ${{ if eq(variables['System.TeamProject'], 'public') }}:
+      - ${{ if and(eq(variables['System.TeamProject'], 'public'), ne(parameters.jobParameters.helixQueueGroup, 'superpmi-diffs')) }}:
         - OSX.13.Arm64.Open
+      - ${{ if and(eq(variables['System.TeamProject'], 'public'), eq(parameters.jobParameters.helixQueueGroup, 'superpmi-diffs')) }}:
+        # superpmi is not about testing platform differences, so go with highest capacity queue
+        - OSX.14.Arm64.Open
       - ${{ if eq(variables['System.TeamProject'], 'internal') }}:
         - OSX.1200.Arm64
 
@@ -122,9 +125,7 @@ jobs:
         - Windows.10.Amd64.Open
       - ${{ if and(eq(variables['System.TeamProject'], 'public'), eq(parameters.jobParameters.helixQueueGroup, 'cet')) }}:
         - Windows.11.Amd64.Cet.Open
-      - ${{ if and(eq(variables['System.TeamProject'], 'internal'), in(parameters.jobParameters.helixQueueGroup, 'superpmi')) }}:
-        - Windows.10.Amd64.X86.Rt
-      - ${{ if and(eq(variables['System.TeamProject'], 'internal'), notIn(parameters.jobParameters.helixQueueGroup, 'superpmi')) }}:
+      - ${{ if eq(variables['System.TeamProject'], 'internal') }}:
         - Windows.10.Amd64
 
     # windows x86

eng/pipelines/coreclr/templates/jit-replay-pipeline.yml

@@ -32,7 +32,7 @@ extends:
           buildConfig: checked
           platforms: ${{ parameters.platforms }}
           jobParameters:
-            buildArgs: -s clr.alljits+clr.spmi -c $(_BuildConfig)
+            buildArgs: -s clr.alljitscommunity+clr.spmi -c $(_BuildConfig)
             postBuildSteps:
               - template: /eng/pipelines/common/upload-artifact-step.yml
                 parameters:

src/coreclr/scripts/superpmi_diffs.py

@@ -111,7 +111,7 @@ def __init__(self, coreclr_args):
         self.spmi_location = os.path.join(coreclr_args.work_directory, "artifacts", "spmi")
 
         self.log_directory = coreclr_args.log_directory
-        self.host_os = "windows" if platform.system() == "Windows" else "linux"
+        self.host_os = CoreclrArguments.provide_default_host_os()
 
         with open(coreclr_args.partition_info, "r") as file:
             partition_info = json.load(file)
@@ -120,7 +120,11 @@ def __init__(self, coreclr_args):
         self.target_os = partition_info["target_os"]
         self.arch_name = partition_info["target_arch"]
         self.col_name = partition_info["col_name"]
-        self.host_arch_name = "x64" if self.arch_name.endswith("64") else "x86"
+
+        if self.arch_name == "arm" or self.arch_name == "x86":
+            self.host_arch_name = "x86"
+        else:
+            self.host_arch_name = CoreclrArguments.provide_default_arch()
 
         # Core_Root is where the superpmi tools (superpmi.exe, mcs.exe) are expected to be found.
         # We pass the full path of the JITs to use as arguments.

src/coreclr/scripts/superpmi_diffs_setup.py

@@ -38,6 +38,7 @@
 parser.add_argument("-release_directory", help="Path to the directory containing built release binaries (e.g., <source_directory>/artifacts/bin/coreclr/windows.x64.Release)")
 
 is_windows = platform.system() == "Windows"
+is_macos = platform.system() == "Darwin"
 target_windows = True
 
 
@@ -111,10 +112,6 @@ def setup_args(args):
             print("release_directory doesn't exist")
             sys.exit(1)
 
-    if coreclr_args.platform.lower() != "windows" and do_asmdiffs:
-        print("asmdiffs currently only implemented for windows")
-        sys.exit(1)
-
     global target_windows
     target_windows = coreclr_args.platform.lower() == "windows"
 
@@ -132,10 +129,10 @@ def match_jit_files(full_path):
     file_name = os.path.basename(full_path)
 
     if target_windows:
-        if file_name.startswith("clrjit_") and file_name.endswith(".dll") and file_name.find("osx") == -1:
+        if file_name.startswith("clrjit_") and file_name.endswith(".dll"):
             return True
     else:
-        if file_name.startswith("libclrjit_") and file_name.endswith(".so") and file_name.find("_win_") == -1 and file_name.find("_arm_") == -1:
+        if file_name.startswith("libclrjit_") and (file_name.endswith(".so") or file_name.endswith(".dylib")):
             return True
 
     return False
@@ -182,10 +179,12 @@ def build_jit_analyze(coreclr_args, source_directory, jit_analyze_build_director
 
             # NOTE: we currently only support running on Windows x86/x64 (we don't pass the target OS)
             RID = None
-            if coreclr_args.arch == "x86":
-                RID = "win-x86"
-            if coreclr_args.arch == "x64":
-                RID = "win-x64"
+            rid_platform = coreclr_args.platform.lower()
+            if rid_platform == "windows":
+                rid_platform = "win"
+
+            rid_arch = coreclr_args.arch
+            RID = f"{rid_platform}-{rid_arch}"
 
             # Set dotnet path to run build
             os.environ["PATH"] = os.path.join(source_directory, ".dotnet") + os.pathsep + os.environ["PATH"]
@@ -203,13 +202,17 @@ def build_jit_analyze(coreclr_args, source_directory, jit_analyze_build_director
         # Details: https://bugs.python.org/issue26660
         print('Ignoring PermissionError: {0}'.format(pe_error))
 
-    jit_analyze_tool = os.path.join(jit_analyze_build_directory, "jit-analyze.exe")
+    jit_analyze_tool = os.path.join(jit_analyze_build_directory, "jit-analyze.exe" if is_windows else "jit-analyze")
     if not os.path.isfile(jit_analyze_tool):
         print('Error: {} not found'.format(jit_analyze_tool))
         return 1
 
-def build_partitions(partitions_dir, bin_path, host_bitness):
+def build_partitions(partitions_dir, do_asmdiffs, bin_path, host_bitness):
     mcs_path = os.path.join(bin_path, "mcs.exe" if is_windows else "mcs")
+    if is_macos:
+        # Hack: the target is arm64, but the build machine is x64. We build SPMI for x64 because of that,
+        # but it exists at a different path.
+        mcs_path = os.path.join(bin_path, "..", "osx.x64.Checked", "mcs")
     assert(os.path.exists(mcs_path))
 
     command = [mcs_path, "-printJITEEVersion"]
@@ -240,7 +243,7 @@ def build_partitions(partitions_dir, bin_path, host_bitness):
 
     elem = ET.fromstring(contents)
 
-    if not target_windows:
+    if not target_windows and not do_asmdiffs:
         targets = [("linux", "x64")]
     elif host_bitness == 64:
         targets = [("windows", "x64"), ("windows", "arm64"), ("linux", "x64"), ("linux", "arm64"), ("osx", "arm64")]
@@ -275,20 +278,20 @@ def main(main_args):
 
     The Helix correlation payload directory is created and populated as follows:
 
-    <source_directory>\payload -- the correlation payload directory
-        -- contains the *.py scripts from <source_directory>\src\coreclr\scripts
+    <source_directory>/payload -- the correlation payload directory
+        -- contains the *.py scripts from <source_directory>/src/coreclr/scripts
         -- contains superpmi.exe, mcs.exe from the target-specific build
-    <source_directory>\payload\base
+    <source_directory>/payload/base
         -- contains the baseline JITs (under checked and release folders)
-    <source_directory>\payload\diff
+    <source_directory>/payload/diff
         -- contains the diff JITs (under checked and release folders)
     For `type == asmdiffs`:
-        <source_directory>\payload\jit-analyze
+        <source_directory>/payload/jit-analyze
             -- contains the self-contained jit-analyze build (from dotnet/jitutils)
-        <source_directory>\payload\git
+        <source_directory>/payload/git
             -- contains a Portable ("xcopy installable") `git` tool, downloaded from:
             https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/git/Git-2.32.0-64-bit.zip
-            This is needed by jit-analyze to do `git diff` on the generated asm. The `<source_directory>\payload\git\cmd`
+            This is needed by jit-analyze to do `git diff` on the generated asm. The `<source_directory>/payload/git/cmd`
             directory is added to the PATH.
             NOTE: this only runs on Windows.
 
@@ -453,7 +456,7 @@ def main(main_args):
     ######## Generate partition information
 
     partitions_dir = os.path.join(correlation_payload_directory, "partitions")
-    build_partitions(partitions_dir, checked_directory if use_checked else release_directory, 64 if coreclr_args.arch == "x64" else 32)
+    build_partitions(partitions_dir, do_asmdiffs, checked_directory if use_checked else release_directory, 64 if coreclr_args.arch in ["x64", "arm64"] else 32)
 
     ######## Set pipeline variables
 

src/coreclr/scripts/superpmi_diffs_summarize.py

@@ -28,8 +28,6 @@
 parser.add_argument("-type", required=True, help="Type of diff (asmdiffs, tpdiff, all)")
 parser.add_argument("-source_directory", required=True, help="Path to the root directory of the dotnet/runtime source tree")
 
-target_windows = True
-
 def setup_args(args):
     """ Setup the args.
 
@@ -68,22 +66,6 @@ def setup_args(args):
                         os.path.isdir,
                         "source_directory doesn't exist")
 
-    do_asmdiffs = False
-    do_tpdiff = False
-    if coreclr_args.type == 'asmdiffs':
-        do_asmdiffs = True
-    if coreclr_args.type == 'tpdiff':
-        do_tpdiff = True
-    if coreclr_args.type == 'all':
-        do_asmdiffs = True
-        do_tpdiff = True
-
-    if coreclr_args.platform.lower() != "windows" and do_asmdiffs:
-        print("asmdiffs currently only implemented for windows")
-        sys.exit(1)
-
-    target_windows = coreclr_args.platform.lower() == "windows"
-
     return coreclr_args