Merge pull request #1862 from folkertdev/combine-c-files

`intrinsic-test`: combine C files for faster compilation

Author: Amanieu

library/stdarch/Cargo.toml

@@ -5,7 +5,8 @@ members = [
   "examples",
 ]
 exclude = [
-  "crates/wasm-assert-instr-tests"
+  "crates/wasm-assert-instr-tests",
+  "rust_programs",
 ]
 
 [profile.release]

library/stdarch/crates/intrinsic-test/src/arm/mod.rs

@@ -4,15 +4,20 @@ mod intrinsic;
 mod json_parser;
 mod types;
 
+use std::fs::File;
+
+use rayon::prelude::*;
+
+use crate::arm::config::POLY128_OSTREAM_DEF;
 use crate::common::SupportedArchitectureTest;
 use crate::common::cli::ProcessedCli;
 use crate::common::compare::compare_outputs;
-use crate::common::gen_c::compile_c_programs;
+use crate::common::gen_c::{write_main_cpp, write_mod_cpp};
 use crate::common::gen_rust::compile_rust_programs;
 use crate::common::intrinsic::{Intrinsic, IntrinsicDefinition};
 use crate::common::intrinsic_helpers::TypeKind;
-use crate::common::write_file::{write_c_testfiles, write_rust_testfiles};
-use config::{AARCH_CONFIGURATIONS, F16_FORMATTING_DEF, POLY128_OSTREAM_DEF, build_notices};
+use crate::common::write_file::write_rust_testfiles;
+use config::{AARCH_CONFIGURATIONS, F16_FORMATTING_DEF, build_notices};
 use intrinsic::ArmIntrinsicType;
 use json_parser::get_neon_intrinsics;
 
@@ -21,6 +26,13 @@ pub struct ArmArchitectureTest {
     cli_options: ProcessedCli,
 }
 
+fn chunk_info(intrinsic_count: usize) -> (usize, usize) {
+    let available_parallelism = std::thread::available_parallelism().unwrap().get();
+    let chunk_size = intrinsic_count.div_ceil(Ord::min(available_parallelism, intrinsic_count));
+
+    (chunk_size, intrinsic_count.div_ceil(chunk_size))
+}
+
 impl SupportedArchitectureTest for ArmArchitectureTest {
     fn create(cli_options: ProcessedCli) -> Box<Self> {
         let a32 = cli_options.target.contains("v7");
@@ -51,24 +63,58 @@ impl SupportedArchitectureTest for ArmArchitectureTest {
     }
 
     fn build_c_file(&self) -> bool {
-        let target = &self.cli_options.target;
         let c_target = "aarch64";
+        let platform_headers = &["arm_neon.h", "arm_acle.h", "arm_fp16.h"];
 
-        let intrinsics_name_list = write_c_testfiles(
-            &self
-                .intrinsics
-                .iter()
-                .map(|i| i as &dyn IntrinsicDefinition<_>)
-                .collect::<Vec<_>>(),
-            target,
+        let (chunk_size, chunk_count) = chunk_info(self.intrinsics.len());
+
+        let cpp_compiler = compile::build_cpp_compilation(&self.cli_options).unwrap();
+
+        let notice = &build_notices("// ");
+        self.intrinsics
+            .par_chunks(chunk_size)
+            .enumerate()
+            .map(|(i, chunk)| {
+                let c_filename = format!("c_programs/mod_{i}.cpp");
+                let mut file = File::create(&c_filename).unwrap();
+                write_mod_cpp(&mut file, notice, c_target, platform_headers, chunk).unwrap();
+
+                // compile this cpp file into a .o file
+                let output = cpp_compiler
+                    .compile_object_file(&format!("mod_{i}.cpp"), &format!("mod_{i}.o"))?;
+                assert!(output.status.success(), "{output:?}");
+
+                Ok(())
+            })
+            .collect::<Result<(), std::io::Error>>()
+            .unwrap();
+
+        let mut file = File::create("c_programs/main.cpp").unwrap();
+        write_main_cpp(
+            &mut file,
             c_target,
-            &["arm_neon.h", "arm_acle.h", "arm_fp16.h"],
-            &build_notices("// "),
-            &[POLY128_OSTREAM_DEF],
-        );
+            POLY128_OSTREAM_DEF,
+            self.intrinsics.iter().map(|i| i.name.as_str()),
+        )
+        .unwrap();
+
+        // compile this cpp file into a .o file
+        info!("compiling main.cpp");
+        let output = cpp_compiler
+            .compile_object_file("main.cpp", "intrinsic-test-programs.o")
+            .unwrap();
+        assert!(output.status.success(), "{output:?}");
+
+        let object_files = (0..chunk_count)
+            .map(|i| format!("mod_{i}.o"))
+            .chain(["intrinsic-test-programs.o".to_owned()]);
+
+        let output = cpp_compiler
+            .link_executable(object_files, "intrinsic-test-programs")
+            .unwrap();
+        assert!(output.status.success(), "{output:?}");
 
-        let pipeline = compile::build_cpp_compilation(&self.cli_options).unwrap();
-        compile_c_programs(&pipeline, &intrinsics_name_list)
+        true
     }
 
     fn build_rust_file(&self) -> bool {

library/stdarch/crates/intrinsic-test/src/common/argument.rs

@@ -125,19 +125,23 @@ where
     /// Creates a line for each argument that initializes an array for C from which `loads` argument
     /// values can be loaded  as a sliding window.
     /// e.g `const int32x2_t a_vals = {0x3effffff, 0x3effffff, 0x3f7fffff}`, if loads=2.
-    pub fn gen_arglists_c(&self, indentation: Indentation, loads: u32) -> String {
-        self.iter()
-            .filter(|&arg| !arg.has_constraint())
-            .map(|arg| {
-                format!(
-                    "{indentation}const {ty} {name}_vals[] = {values};",
-                    ty = arg.ty.c_scalar_type(),
-                    name = arg.name,
-                    values = arg.ty.populate_random(indentation, loads, &Language::C)
-                )
-            })
-            .collect::<Vec<_>>()
-            .join("\n")
+    pub fn gen_arglists_c(
+        &self,
+        w: &mut impl std::io::Write,
+        indentation: Indentation,
+        loads: u32,
+    ) -> std::io::Result<()> {
+        for arg in self.iter().filter(|&arg| !arg.has_constraint()) {
+            writeln!(
+                w,
+                "{indentation}const {ty} {name}_vals[] = {values};",
+                ty = arg.ty.c_scalar_type(),
+                name = arg.name,
+                values = arg.ty.populate_random(indentation, loads, &Language::C)
+            )?
+        }
+
+        Ok(())
     }
 
     /// Creates a line for each argument that initializes an array for Rust from which `loads` argument

library/stdarch/crates/intrinsic-test/src/common/compare.rs

@@ -15,12 +15,12 @@ pub fn compare_outputs(intrinsic_name_list: &Vec<String>, runner: &str, target:
         .par_iter()
         .filter_map(|intrinsic_name| {
             let c = runner_command(runner)
-                .arg(format!("./c_programs/{intrinsic_name}"))
+                .arg("./c_programs/intrinsic-test-programs")
+                .arg(intrinsic_name)
                 .output();
 
             let rust = runner_command(runner)
                 .arg(format!("target/{target}/release/{intrinsic_name}"))
-                .env("RUSTFLAGS", "-Cdebuginfo=0")
                 .output();
 
             let (c, rust) = match (c, rust) {

library/stdarch/crates/intrinsic-test/src/common/compile_c.rs

@@ -112,11 +112,24 @@ impl CppCompilation {
         &mut self.0
     }
 
-    pub fn run(&self, inputs: &[String], output: &str) -> std::io::Result<std::process::Output> {
+    pub fn compile_object_file(
+        &self,
+        input: &str,
+        output: &str,
+    ) -> std::io::Result<std::process::Output> {
+        let mut cmd = clone_command(&self.0);
+        cmd.args([input, "-c", "-o", output]);
+        cmd.output()
+    }
+
+    pub fn link_executable(
+        &self,
+        inputs: impl Iterator<Item = String>,
+        output: &str,
+    ) -> std::io::Result<std::process::Output> {
         let mut cmd = clone_command(&self.0);
         cmd.args(inputs);
         cmd.args(["-o", output]);
-
         cmd.output()
     }
 }