Cleanup

Author: kunalspathak

eng/pipelines/coreclr/superpmi-replay.yml

@@ -20,16 +20,16 @@ jobs:
     buildConfig: checked
     platforms:
     - windows_x64
-    # - windows_x86
+    - windows_x86
     jobParameters:
       uploadAsArtifacts: true
 
-# - template: /eng/pipelines/common/platform-matrix.yml
-#   parameters:
-#     jobTemplate: /eng/pipelines/coreclr/templates/superpmi-replay-job.yml
-#     buildConfig: checked
-#     platforms:
-#     - windows_x64
-#     - windows_x86
-#     helixQueueGroup: ci
-#     helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
+- template: /eng/pipelines/common/platform-matrix.yml
+  parameters:
+    jobTemplate: /eng/pipelines/coreclr/templates/superpmi-replay-job.yml
+    buildConfig: checked
+    platforms:
+    - windows_x64
+    - windows_x86
+    helixQueueGroup: ci
+    helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml

eng/pipelines/coreclr/templates/build-jit-job.yml

@@ -81,9 +81,6 @@ jobs:
     - ${{ parameters.variables }}
 
     steps:
-    - script: |
-        echo $(uploadAsArtifacts)
-        echo ${{ parameters.uploadAsArtifacts }}
 
     # Install native dependencies
     # Linux builds use docker images with dependencies preinstalled,

src/coreclr/scripts/superpmi-replay.proj

@@ -11,7 +11,6 @@
 
    <!-- <PropertyGroup>
      <HelixTargetQueues>Some_Queue</HelixTargetQueues>
-     <WorkItemDirectory>E:\temp\replay-setup\workitem</WorkItemDirectory>
      <Platform>Windows</Platform>
      <Architecture>x64</Architecture>
    </PropertyGroup>
@@ -45,14 +44,14 @@
 
   <ItemGroup Condition="'$(Architecture)' == 'x64'">
     <SPMI_Partition Include="win-x64" Platform="windows" Architecture="x64" />
-    <!-- <SPMI_Partition Include="win-arm64" Platform="windows" Architecture="arm64" />
+    <SPMI_Partition Include="win-arm64" Platform="windows" Architecture="arm64" />
     <SPMI_Partition Include="unix-x64" Platform="Linux" Architecture="x64" />
-    <SPMI_Partition Include="unix-arm64" Platform="Linux" Architecture="arm64" /> -->
+    <SPMI_Partition Include="unix-arm64" Platform="Linux" Architecture="arm64" />
   </ItemGroup>
 
   <ItemGroup Condition="'$(Architecture)' == 'x86'">
     <SPMI_Partition Include="win-x86" Platform="windows" Architecture="x86" />
-    <!-- <SPMI_Partition Include="unix-arm" Platform="Linux" Architecture="arm" /> -->
+    <SPMI_Partition Include="unix-arm" Platform="Linux" Architecture="arm" />
   </ItemGroup>
 
   <ItemGroup>

src/coreclr/scripts/superpmi-replay.py

@@ -1,3 +1,17 @@
+#!/usr/bin/env python3
+#
+# Licensed to the .NET Foundation under one or more agreements.
+# The .NET Foundation licenses this file to you under the MIT license.
+#
+##
+# Title               : superpmi_setup.py
+#
+# Notes:
+#
+# Script to run "superpmi replay" for various collections under various COMPlus_JitStressRegs value.
+################################################################################
+################################################################################
+
 
 import argparse
 from os import path
@@ -23,7 +37,8 @@
     "JitStressRegs=0x10",
     "JitStressRegs=0x80",
     "JitStressRegs=0x1000",
-    ]
+]
+
 
 def setup_args(args):
     """ Setup the args for SuperPMI to use.
@@ -60,6 +75,7 @@ def setup_args(args):
 
     return coreclr_args
 
+
 def main(main_args):
     """Main entrypoint
 
@@ -70,32 +86,30 @@ def main(main_args):
     python_path = sys.executable
     cwd = os.path.dirname(os.path.realpath(__file__))
     coreclr_args = setup_args(main_args)
-    # mch_directory = coreclr_args.mch_directory
     spmi_location = path.join(cwd, "artifacts", "spmi")
     log_directory = coreclr_args.log_directory
-    mch_filename = ''
     platform_name = coreclr_args.platform
     os_name = "win" if platform_name.lower() == "windows" else "unix"
     arch_name = coreclr_args.arch
     host_arch_name = "x64" if arch_name.endswith("64") else "x86"
     jit_path = path.join(coreclr_args.jit_directory, 'clrjit_{}_{}_{}.dll'.format(os_name, arch_name, host_arch_name))
 
     print("Running superpmi.py download")
-    run_command([python_path, path.join(cwd, "superpmi.py"), "download", "--no_progress", "-f", "benchmark", "-target_os", platform_name, "-target_arch", arch_name, "-core_root", cwd, "-spmi_location", spmi_location],  _exit_on_fail=True)
+    run_command([python_path, path.join(cwd, "superpmi.py"), "download", "--no_progress", "-target_os", platform_name,
+                 "-target_arch", arch_name, "-core_root", cwd, "-spmi_location", spmi_location], _exit_on_fail=True)
 
     failed_runs = []
     for jit_flag in jit_flags:
-        # TODO: This should be DownloadFilesFromResults
         log_file = path.join(log_directory, 'superpmi_{}.log'.format(jit_flag.replace("=", "_")))
         print("Running superpmi.py replay for {}".format(jit_flag))
 
         _, _, return_code = run_command([
-                python_path, path.join(cwd, "superpmi.py"), "replay", "-core_root", cwd,
-                "-jitoption", jit_flag, "-jitoption", "TieredCompilation=0",
-                "-target_os", platform_name, "-target_arch", arch_name,
-                "-arch", host_arch_name,
-                "-jit_path", jit_path, "-spmi_location", spmi_location,
-                "-log_level", "debug", "-log_file", log_file])
+            python_path, path.join(cwd, "superpmi.py"), "replay", "-core_root", cwd,
+            "-jitoption", jit_flag, "-jitoption", "TieredCompilation=0",
+            "-target_os", platform_name, "-target_arch", arch_name,
+            "-arch", host_arch_name,
+            "-jit_path", jit_path, "-spmi_location", spmi_location,
+            "-log_level", "debug", "-log_file", log_file])
 
         if return_code != 0:
             failed_runs.append("Failure in {}".format(log_file))
@@ -116,6 +130,7 @@ def main(main_args):
                 contents = current_superpmi_log.read()
                 final_superpmi_log.write(contents)
 
+        # Log failures summary
         if len(failed_runs) > 0:
             final_superpmi_log.write(os.linesep)
             final_superpmi_log.write(os.linesep)

src/coreclr/scripts/superpmi_replay_setup.py

@@ -9,11 +9,7 @@
 # Notes:
 #
 # Script to setup directory structure required to perform SuperPMI replay in CI.
-# It does the following steps:
-# 1. It creates `correlation_payload_directory` that contains clrjit*_x64.dll and clrjit*_x86.dll
-# 2. The script takes `input_artifacts` parameter which contains *.mch.zip and *.mct.zip. It will then
-#    partition it by moving each pair of *.mch.zip/*.mct.zip into its own folder under 'payload'
-#    directory.
+#  It creates `correlation_payload_directory` that contains clrjit*_x64.dll and clrjit*_x86.dll
 ################################################################################
 ################################################################################
 
@@ -32,10 +28,8 @@
 parser = argparse.ArgumentParser(description="description")
 
 parser.add_argument("-arch", help="Architecture")
-# parser.add_argument("-platform", help="OS platform")
 parser.add_argument("-source_directory", help="path to the directory containing binaries")
 parser.add_argument("-product_directory", help="path to the directory containing binaries")
-# parser.add_argument("-mch_directory", help="path to directory containing compressed mch files")
 
 
 def setup_args(args):
@@ -56,11 +50,6 @@ def setup_args(args):
                         lambda unused: True,
                         "Unable to set arch")
 
-    # coreclr_args.verify(args,
-    #                     "platform",
-    #                     lambda unused: True,
-    #                     "Unable to set platform")
-
     coreclr_args.verify(args,
                         "source_directory",
                         lambda source_directory: os.path.isdir(source_directory),
@@ -71,10 +60,6 @@ def setup_args(args):
                         lambda product_directory: os.path.isdir(product_directory),
                         "product_directory doesn't exist")
 
-    # coreclr_args.verify(args,
-    #                     "mch_directory",
-    #                     lambda mch_directory: True, #os.path.isdir(mch_directory),
-    #                     "mch_directory doesn't exist")
     return coreclr_args
 
 
@@ -117,6 +102,7 @@ def match_correlation_files(full_path):
 
     return False
 
+
 def main(main_args):
     """Main entrypoint
 
@@ -128,58 +114,34 @@ def main(main_args):
     arch = coreclr_args.arch
     source_directory = coreclr_args.source_directory
     product_directory = coreclr_args.product_directory
-    # mch_directory = coreclr_args.mch_directory
 
     # CorrelationPayload directories
     correlation_payload_directory = path.join(coreclr_args.source_directory, "payload")
     superpmi_src_directory = path.join(source_directory, 'src', 'coreclr', 'scripts')
 
-    # Workitem directories
-    workitem_directory = path.join(source_directory, "workitem")
-
     helix_source_prefix = "official"
     creator = ""
     ci = True
     helix_queue = "Windows.10.Amd64.X86"
 
     # Copy *.py to CorrelationPayload
     print('Copying {} -> {}'.format(superpmi_src_directory, correlation_payload_directory))
-    copy_directory(superpmi_src_directory, correlation_payload_directory, match_func=lambda path: any(path.endswith(extension) for extension in [".py"]))
-    
+    copy_directory(superpmi_src_directory, correlation_payload_directory,
+                   match_func=lambda path: any(path.endswith(extension) for extension in [".py"]))
+
     # Copy clrjit*_arch.dll binaries to CorrelationPayload
     print('Copying binaries {} -> {}'.format(arch, product_directory, correlation_payload_directory))
     copy_directory(product_directory, correlation_payload_directory, match_func=match_correlation_files)
 
-    #TODO: Just send appropriate clrjit*.dll files to workitem_directory
-    # # Copy clrjit*_arch.dll binaries to workitem_directory
-    # print('Copying clrjit_{}_{}.dll {} -> {}'.format(arch, product_directory, correlation_payload_directory))
-    # copy_directory(product_directory, correlation_payload_directory, match_func=match_correlation_files)
-
-    # Partition mch/mct zip files
-    # partition_mch(mch_directory, workitem_directory)
-
-    # # Print correlation_payload_directory and workitem_directory
-    # print("==> correlation_payload_directory:")
-    # for file_path, dirs, files in walk(correlation_payload_directory, topdown=True):
-    #     for name in files:
-    #         curr_file_path = path.join(file_path, name)
-    #         print(curr_file_path)
-
-    # print("==> workitem_directory:")
-    # for file_path, dirs, files in walk(workitem_directory, topdown=True):
-    #     for name in files:
-    #         curr_file_path = path.join(file_path, name)
-    #         print(curr_file_path)
-
     # Set variables
     print('Setting pipeline variables:')
     set_pipeline_variable("CorrelationPayloadDirectory", correlation_payload_directory)
-    set_pipeline_variable("WorkItemDirectory", workitem_directory)
     set_pipeline_variable("Architecture", arch)
     set_pipeline_variable("Creator", creator)
     set_pipeline_variable("Queue", helix_queue)
     set_pipeline_variable("HelixSourcePrefix", helix_source_prefix)
 
+
 if __name__ == "__main__":
     args = parser.parse_args()
     sys.exit(main(args))