Using .ilc.rsp files for superpmi.py nativeaot

src/coreclr/scripts/superpmi.py

@@ -861,9 +861,17 @@ def set_and_report_env(env, root_env, dotnet_env = None):
             # If we need them, collect all the assemblies we're going to use for the collection(s).
             # Remove the files matching the `-exclude` arguments (case-insensitive) from the list.
             if self.coreclr_args.pmi or self.coreclr_args.crossgen2 or self.coreclr_args.nativeaot:
+
+                def filter_assembly(file):
+                    if self.coreclr_args.nativeaot:
+                        extensions = [".ilc.rsp"]
+                    else:
+                        extensions = [".dll", ".exe"]
+                    return any(file.endswith(extension) for extension in extensions) and (self.exclude is None or not any(e.lower() in file.lower() for e in self.exclude))
+
                 assemblies = []
                 for item in self.assemblies:
-                    assemblies += get_files_from_path(item, match_func=lambda file: any(file.endswith(extension) for extension in [".dll", ".exe"]) and (self.exclude is None or not any(e.lower() in file.lower() for e in self.exclude)))
+                    assemblies += get_files_from_path(item, match_func=filter_assembly)
                 if len(assemblies) == 0:
                     logging.error("No assemblies found using `-assemblies` and `-exclude` arguments!")
                 else:
@@ -1114,51 +1122,9 @@ async def run_crossgen2(print_prefix, assembly, self):
             if self.coreclr_args.nativeaot is True:
                 logging.debug("Starting collection using nativeaot")
 
-                async def run_nativeaot(print_prefix, assembly, self):
-                    """ Run nativeaot over all dlls
-                    """
-
-                    root_nativeaot_output_filename = make_safe_filename("nativeaot_" + assembly) + ".out.dll"
-                    nativeaot_output_assembly_filename = os.path.join(self.temp_location, root_nativeaot_output_filename)
-                    try:
-                        if os.path.exists(nativeaot_output_assembly_filename):
-                            os.remove(nativeaot_output_assembly_filename)
-                    except OSError as ose:
-                        if "[WinError 32] The process cannot access the file because it is being used by another " \
-                           "process:" in format(ose):
-                            logging.warning("Skipping file %s. Got error: %s", nativeaot_output_assembly_filename, ose)
-                            return
-                        else:
-                            raise ose
-
-                    root_output_filename = make_safe_filename("nativeaot_" + assembly + "_")
-
-                    # Create a temporary response file to put all the arguments to nativeaot (otherwise the path length limit could be exceeded):
-                    #
-                    # <dll to compile>
-                    # -o:<output dll>
-                    # -r:<Core_Root>\System.*.dll
-                    # -r:<Core_Root>\Microsoft.*.dll
-                    # -r:<Core_Root>\System.Private.CoreLib.dll
-                    # -r:<Core_Root>\netstandard.dll
-                    # --jitpath:<self.collection_shim_name>
-                    # --codegenopt:<option>=<value>   /// for each member of dotnet_env
-                    #
-                    # invoke with:
-                    #
-                    # ilc.exe @<temp.rsp>
-                    #
-
-                    rsp_file_handle, rsp_filepath = tempfile.mkstemp(suffix=".rsp", prefix=root_output_filename, dir=self.temp_location)
-                    with open(rsp_file_handle, "w") as rsp_write_handle:
-                        rsp_write_handle.write(assembly + "\n")
-                        rsp_write_handle.write("-o:" + nativeaot_output_assembly_filename + "\n")
-                        rsp_write_handle.write("-r:" + os.path.join(self.coreclr_args.nativeaot_aotsdk_path, "System.*.dll") + "\n")
-                        rsp_write_handle.write("-r:" + os.path.join(self.core_root, "System.*.dll") + "\n")
-                        rsp_write_handle.write("-r:" + os.path.join(self.core_root, "Microsoft.*.dll") + "\n")
-                        rsp_write_handle.write("-r:" + os.path.join(self.core_root, "mscorlib.dll") + "\n")
-                        rsp_write_handle.write("-r:" + os.path.join(self.core_root, "netstandard.dll") + "\n")
-                        rsp_write_handle.write("--parallelism:1" + "\n")
+                async def run_nativeaot(print_prefix, rsp_filepath, self):
+                    rsp_filepath = assembly
+                    with open(rsp_filepath, "a") as rsp_write_handle:
                         rsp_write_handle.write("--jitpath:" + os.path.join(self.core_root, self.collection_shim_name) + "\n")
                         for var, value in dotnet_env.items():
                             rsp_write_handle.write("--codegenopt:" + var + "=" + value + "\n")
@@ -1172,6 +1138,8 @@ async def run_nativeaot(print_prefix, assembly, self):
 
                     begin_time = datetime.datetime.now()
 
+                    root_output_filename = assembly
+
                     # Save the stdout and stderr to files, so we can see if nativeaot wrote any interesting messages.
                     # Use the name of the assembly as the basename of the file. mkstemp() will ensure the file
                     # is unique.
@@ -1222,11 +1190,7 @@ async def run_nativeaot(print_prefix, assembly, self):
                 old_env = os.environ.copy()
                 os.environ.update(nativeaot_command_env)
 
-                # Note: nativeaot compiles in parallel by default. However, it seems to lead to sharing violations
-                # in SuperPMI collection, accessing the MC file. So, disable nativeaot parallism by using
-                # the "--parallelism:1" switch, and allowing coarse-grained (per-assembly) parallelism here.
-                # It turns out this works better anyway, as there is a lot of non-parallel time between
-                # nativeaot parallel compilations.
+                assemblies = list(filter(lambda assembly: assembly.endswith(".ilc.rsp"), assemblies))
                 helper = AsyncSubprocessHelper(assemblies, verbose=True)
                 helper.run_to_completion(run_nativeaot, self)
 

src/coreclr/scripts/superpmi_collect_setup.py

@@ -302,7 +302,7 @@ def sorter_by_size(pair):
 
             if not os.path.isfile(curr_file_path):
                 continue
-            if not name.endswith(".dll") and not name.endswith(".exe"):
+            if not name.endswith(".dll") and not name.endswith(".exe") and not name.endswith(".ilc.rsp"):
                 continue
 
             size = os.path.getsize(curr_file_path)
@@ -558,9 +558,9 @@ def make_readable(folder_name):
                 print('Ignoring PermissionError: {0}'.format(pe_error))
 
         # Build nativeaot tests
-        # if coreclr_args.collection_type == "nativeaot":
-        #     tests_build_file = "build.cmd" if is_windows else "build.sh"
-        #     run_command([os.path.join(tests_directory, tests_build_file), "nativeaot", coreclr_args.build_type, "tree", "nativeaot"], source_directory)
+        if coreclr_args.collection_type == "nativeaot":
+            tests_build_file = "build.cmd" if is_windows else "build.sh"
+            run_command([os.path.join(tests_directory, tests_build_file), "nativeaot", coreclr_args.build_type, "tree", "nativeaot"], source_directory)
 
         # NOTE: we can't use the build machine ".dotnet" to run on all platforms. E.g., the Windows x86 build uses a
         # Windows x64 .dotnet\dotnet.exe that can't load a 32-bit shim. Thus, we always use corerun from Core_Root to invoke crossgen2.
@@ -591,11 +591,6 @@ def make_readable(folder_name):
             exclude_files += [item for item in os.listdir(core_root_dir)
                               if os.path.isfile(os.path.join(core_root_dir, item)) and (item.endswith(".dll") or item.endswith(".exe"))]
 
-        if coreclr_args.collection_type == "nativeaot":
-            # do not include the test wrappers
-            exclude_files += ["Coreclr.TestWrapper.dll", "nativeaot.SmokeTests.XUnitWrapper.dll"]
-            exclude_directories += ['native']
-
         partition_files(coreclr_args.input_directory, input_artifacts, coreclr_args.max_size, exclude_directories,
                         exclude_files)