Merge commit '8f9ac9c22d6594cf059d8e6c71d414cc5ccd7975' into sync_cg_…

…clif-2023-08-09

.github/workflows/audit.yml

@@ -0,0 +1,19 @@
+name: Security audit
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 10 * * 1' # every monday at 10:00 UTC
+permissions:
+  issues: write
+  checks: write
+jobs:
+  audit:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - run: |
+          sed -i 's/components.*/components = []/' rust-toolchain
+          echo 'profile = "minimal"' >> rust-toolchain
+      - uses: rustsec/audit-check@v1.4.1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}

build_system/abi_cafe.rs

@@ -1,8 +1,8 @@
-use super::build_sysroot;
-use super::path::Dirs;
-use super::prepare::GitRepo;
-use super::utils::{spawn_and_wait, CargoProject, Compiler};
-use super::{CodegenBackend, SysrootKind};
+use crate::build_sysroot;
+use crate::path::Dirs;
+use crate::prepare::GitRepo;
+use crate::utils::{spawn_and_wait, CargoProject, Compiler};
+use crate::{CodegenBackend, SysrootKind};
 
 static ABI_CAFE_REPO: GitRepo = GitRepo::github(
     "Gankra",

build_system/bench.rs

@@ -2,10 +2,10 @@ use std::env;
 use std::io::Write;
 use std::path::Path;
 
-use super::path::{Dirs, RelPath};
-use super::prepare::GitRepo;
-use super::rustc_info::get_file_name;
-use super::utils::{hyperfine_command, spawn_and_wait, Compiler};
+use crate::path::{Dirs, RelPath};
+use crate::prepare::GitRepo;
+use crate::rustc_info::get_file_name;
+use crate::utils::{hyperfine_command, spawn_and_wait, Compiler};
 
 static SIMPLE_RAYTRACER_REPO: GitRepo = GitRepo::github(
     "ebobby",

build_system/build_backend.rs

@@ -1,9 +1,9 @@
-use std::env;
 use std::path::PathBuf;
 
-use super::path::{Dirs, RelPath};
-use super::rustc_info::get_file_name;
-use super::utils::{is_ci, is_ci_opt, maybe_incremental, CargoProject, Compiler, LogGroup};
+use crate::path::{Dirs, RelPath};
+use crate::rustc_info::get_file_name;
+use crate::shared_utils::{rustflags_from_env, rustflags_to_cmd_env};
+use crate::utils::{is_ci, is_ci_opt, maybe_incremental, CargoProject, Compiler, LogGroup};
 
 pub(crate) static CG_CLIF: CargoProject = CargoProject::new(&RelPath::SOURCE, "cg_clif");
 
@@ -18,11 +18,11 @@ pub(crate) fn build_backend(
     let mut cmd = CG_CLIF.build(&bootstrap_host_compiler, dirs);
     maybe_incremental(&mut cmd);
 
-    let mut rustflags = env::var("RUSTFLAGS").unwrap_or_default();
+    let mut rustflags = rustflags_from_env("RUSTFLAGS");
 
     if is_ci() {
         // Deny warnings on CI
-        rustflags += " -Dwarnings";
+        rustflags.push("-Dwarnings".to_owned());
 
         if !is_ci_opt() {
             cmd.env("CARGO_PROFILE_RELEASE_DEBUG_ASSERTIONS", "true");
@@ -42,10 +42,10 @@ pub(crate) fn build_backend(
         _ => unreachable!(),
     }
 
-    cmd.env("RUSTFLAGS", rustflags);
+    rustflags_to_cmd_env(&mut cmd, "RUSTFLAGS", &rustflags);
 
     eprintln!("[BUILD] rustc_codegen_cranelift");
-    super::utils::spawn_and_wait(cmd);
+    crate::utils::spawn_and_wait(cmd);
 
     CG_CLIF
         .target_dir(dirs)

build_system/build_sysroot.rs

@@ -2,13 +2,13 @@ use std::fs;
 use std::path::{Path, PathBuf};
 use std::process::Command;
 
-use super::path::{Dirs, RelPath};
-use super::rustc_info::get_file_name;
-use super::utils::{
+use crate::path::{Dirs, RelPath};
+use crate::rustc_info::get_file_name;
+use crate::utils::{
     maybe_incremental, remove_dir_if_exists, spawn_and_wait, try_hard_link, CargoProject, Compiler,
     LogGroup,
 };
-use super::{CodegenBackend, SysrootKind};
+use crate::{config, CodegenBackend, SysrootKind};
 
 static DIST_DIR: RelPath = RelPath::DIST;
 static BIN_DIR: RelPath = RelPath::DIST.join("bin");
@@ -128,8 +128,8 @@ pub(crate) fn build_sysroot(
             cargo: bootstrap_host_compiler.cargo.clone(),
             rustc: rustc_clif.clone(),
             rustdoc: rustdoc_clif.clone(),
-            rustflags: String::new(),
-            rustdocflags: String::new(),
+            rustflags: vec![],
+            rustdocflags: vec![],
             triple: target_triple,
             runner: vec![],
         }
@@ -185,7 +185,7 @@ fn build_sysroot_for_triple(
 
 #[must_use]
 fn build_llvm_sysroot_for_triple(compiler: Compiler) -> SysrootTarget {
-    let default_sysroot = super::rustc_info::get_default_sysroot(&compiler.rustc);
+    let default_sysroot = crate::rustc_info::get_default_sysroot(&compiler.rustc);
 
     let mut target_libs = SysrootTarget { triple: compiler.triple, libs: vec![] };
 
@@ -234,32 +234,32 @@ fn build_clif_sysroot_for_triple(
 
     let build_dir = STANDARD_LIBRARY.target_dir(dirs).join(&compiler.triple).join(channel);
 
-    if !super::config::get_bool("keep_sysroot") {
+    if !config::get_bool("keep_sysroot") {
         // Cleanup the deps dir, but keep build scripts and the incremental cache for faster
         // recompilation as they are not affected by changes in cg_clif.
         remove_dir_if_exists(&build_dir.join("deps"));
     }
 
     // Build sysroot
-    let mut rustflags = " -Zforce-unstable-if-unmarked -Cpanic=abort".to_string();
+    let mut rustflags = vec!["-Zforce-unstable-if-unmarked".to_owned(), "-Cpanic=abort".to_owned()];
     match cg_clif_dylib_path {
         CodegenBackend::Local(path) => {
-            rustflags.push_str(&format!(" -Zcodegen-backend={}", path.to_str().unwrap()));
+            rustflags.push(format!("-Zcodegen-backend={}", path.to_str().unwrap()));
         }
         CodegenBackend::Builtin(name) => {
-            rustflags.push_str(&format!(" -Zcodegen-backend={name}"));
+            rustflags.push(format!("-Zcodegen-backend={name}"));
         }
     };
     // Necessary for MinGW to find rsbegin.o and rsend.o
-    rustflags
-        .push_str(&format!(" --sysroot {}", RTSTARTUP_SYSROOT.to_path(dirs).to_str().unwrap()));
+    rustflags.push("--sysroot".to_owned());
+    rustflags.push(RTSTARTUP_SYSROOT.to_path(dirs).to_str().unwrap().to_owned());
     if channel == "release" {
         // Incremental compilation by default disables mir inlining. This leads to both a decent
         // compile perf and a significant runtime perf regression. As such forcefully enable mir
         // inlining.
-        rustflags.push_str(" -Zinline-mir");
+        rustflags.push("-Zinline-mir".to_owned());
     }
-    compiler.rustflags += &rustflags;
+    compiler.rustflags.extend(rustflags);
     let mut build_cmd = STANDARD_LIBRARY.build(&compiler, dirs);
     maybe_incremental(&mut build_cmd);
     if channel == "release" {
@@ -289,8 +289,8 @@ fn build_clif_sysroot_for_triple(
 }
 
 fn build_rtstartup(dirs: &Dirs, compiler: &Compiler) -> Option<SysrootTarget> {
-    if !super::config::get_bool("keep_sysroot") {
-        super::prepare::prepare_stdlib(dirs, &compiler.rustc);
+    if !config::get_bool("keep_sysroot") {
+        crate::prepare::prepare_stdlib(dirs, &compiler.rustc);
     }
 
     if !compiler.triple.ends_with("windows-gnu") {
@@ -306,6 +306,7 @@ fn build_rtstartup(dirs: &Dirs, compiler: &Compiler) -> Option<SysrootTarget> {
         let obj = RTSTARTUP_SYSROOT.to_path(dirs).join(format!("{file}.o"));
         let mut build_rtstartup_cmd = Command::new(&compiler.rustc);
         build_rtstartup_cmd
+            .arg("-Ainternal_features") // Missing #[allow(internal_features)]
             .arg("--target")
             .arg(&compiler.triple)
             .arg("--emit=obj")

build_system/main.rs

@@ -16,6 +16,7 @@ mod config;
 mod path;
 mod prepare;
 mod rustc_info;
+mod shared_utils;
 mod tests;
 mod utils;
 
@@ -169,8 +170,8 @@ fn main() {
             cargo,
             rustc,
             rustdoc,
-            rustflags: String::new(),
-            rustdocflags: String::new(),
+            rustflags: vec![],
+            rustdocflags: vec![],
             triple,
             runner: vec![],
         }

build_system/path.rs

@@ -1,7 +1,7 @@
 use std::fs;
 use std::path::PathBuf;
 
-use super::utils::remove_dir_if_exists;
+use crate::utils::remove_dir_if_exists;
 
 #[derive(Debug, Clone)]
 pub(crate) struct Dirs {

build_system/prepare.rs

@@ -3,18 +3,18 @@ use std::fs;
 use std::path::{Path, PathBuf};
 use std::process::Command;
 
-use super::build_sysroot::STDLIB_SRC;
-use super::path::{Dirs, RelPath};
-use super::rustc_info::get_default_sysroot;
-use super::utils::{
+use crate::build_sysroot::STDLIB_SRC;
+use crate::path::{Dirs, RelPath};
+use crate::rustc_info::get_default_sysroot;
+use crate::utils::{
     copy_dir_recursively, git_command, remove_dir_if_exists, retry_spawn_and_wait, spawn_and_wait,
 };
 
 pub(crate) fn prepare(dirs: &Dirs) {
     RelPath::DOWNLOAD.ensure_exists(dirs);
-    super::tests::RAND_REPO.fetch(dirs);
-    super::tests::REGEX_REPO.fetch(dirs);
-    super::tests::PORTABLE_SIMD_REPO.fetch(dirs);
+    crate::tests::RAND_REPO.fetch(dirs);
+    crate::tests::REGEX_REPO.fetch(dirs);
+    crate::tests::PORTABLE_SIMD_REPO.fetch(dirs);
 }
 
 pub(crate) fn prepare_stdlib(dirs: &Dirs, rustc: &Path) {

build_system/shared_utils.rs

@@ -0,0 +1,26 @@
+// This file is used by both the build system as well as cargo-clif.rs
+
+// Adapted from https://github.com/rust-lang/cargo/blob/6dc1deaddf62c7748c9097c7ea88e9ec77ff1a1a/src/cargo/core/compiler/build_context/target_info.rs#L750-L77
+pub(crate) fn rustflags_from_env(kind: &str) -> Vec<String> {
+    // First try CARGO_ENCODED_RUSTFLAGS from the environment.
+    // Prefer this over RUSTFLAGS since it's less prone to encoding errors.
+    if let Ok(a) = std::env::var(format!("CARGO_ENCODED_{}", kind)) {
+        if a.is_empty() {
+            return Vec::new();
+        }
+        return a.split('\x1f').map(str::to_string).collect();
+    }
+
+    // Then try RUSTFLAGS from the environment
+    if let Ok(a) = std::env::var(kind) {
+        let args = a.split(' ').map(str::trim).filter(|s| !s.is_empty()).map(str::to_string);
+        return args.collect();
+    }
+
+    // No rustflags to be collected from the environment
+    Vec::new()
+}
+
+pub(crate) fn rustflags_to_cmd_env(cmd: &mut std::process::Command, kind: &str, flags: &[String]) {
+    cmd.env(format!("CARGO_ENCODED_{}", kind), flags.join("\x1f"));
+}

build_system/tests.rs

@@ -1,16 +1,17 @@
-use super::build_sysroot;
-use super::config;
-use super::path::{Dirs, RelPath};
-use super::prepare::{apply_patches, GitRepo};
-use super::rustc_info::get_default_sysroot;
-use super::utils::{spawn_and_wait, spawn_and_wait_with_input, CargoProject, Compiler, LogGroup};
-use super::{CodegenBackend, SysrootKind};
-use std::env;
 use std::ffi::OsStr;
 use std::fs;
 use std::path::PathBuf;
 use std::process::Command;
 
+use crate::build_sysroot;
+use crate::config;
+use crate::path::{Dirs, RelPath};
+use crate::prepare::{apply_patches, GitRepo};
+use crate::rustc_info::get_default_sysroot;
+use crate::shared_utils::rustflags_from_env;
+use crate::utils::{spawn_and_wait, spawn_and_wait_with_input, CargoProject, Compiler, LogGroup};
+use crate::{CodegenBackend, SysrootKind};
+
 static BUILD_EXAMPLE_OUT_DIR: RelPath = RelPath::BUILD.join("example");
 
 struct TestCase {
@@ -306,7 +307,7 @@ pub(crate) fn run_tests(
         );
         // Rust's build system denies a couple of lints that trigger on several of the test
         // projects. Changing the code to fix them is not worth it, so just silence all lints.
-        target_compiler.rustflags += " --cap-lints=allow";
+        target_compiler.rustflags.push("--cap-lints=allow".to_owned());
 
         let runner = TestRunner::new(
             dirs.clone(),
@@ -350,18 +351,15 @@ impl<'a> TestRunner<'a> {
         is_native: bool,
         stdlib_source: PathBuf,
     ) -> Self {
-        if let Ok(rustflags) = env::var("RUSTFLAGS") {
-            target_compiler.rustflags.push(' ');
-            target_compiler.rustflags.push_str(&rustflags);
-        }
-        if let Ok(rustdocflags) = env::var("RUSTDOCFLAGS") {
-            target_compiler.rustdocflags.push(' ');
-            target_compiler.rustdocflags.push_str(&rustdocflags);
-        }
+        target_compiler.rustflags.extend(rustflags_from_env("RUSTFLAGS"));
+        target_compiler.rustdocflags.extend(rustflags_from_env("RUSTDOCFLAGS"));
 
         // FIXME fix `#[linkage = "extern_weak"]` without this
         if target_compiler.triple.contains("darwin") {
-            target_compiler.rustflags.push_str(" -Clink-arg=-undefined -Clink-arg=dynamic_lookup");
+            target_compiler.rustflags.extend([
+                "-Clink-arg=-undefined".to_owned(),
+                "-Clink-arg=dynamic_lookup".to_owned(),
+            ]);
         }
 
         let jit_supported = use_unstable_features
@@ -470,7 +468,7 @@ impl<'a> TestRunner<'a> {
         S: AsRef<OsStr>,
     {
         let mut cmd = Command::new(&self.target_compiler.rustc);
-        cmd.args(self.target_compiler.rustflags.split_whitespace());
+        cmd.args(&self.target_compiler.rustflags);
         cmd.arg("-L");
         cmd.arg(format!("crate={}", BUILD_EXAMPLE_OUT_DIR.to_path(&self.dirs).display()));
         cmd.arg("--out-dir");

build_system/usage.txt

@@ -43,7 +43,7 @@ REQUIREMENTS:
     * Rustup: By default rustup is used to install the right nightly version. If you don't want to
       use rustup, you can manually install the nightly version indicated by rust-toolchain.toml and
       point the CARGO, RUSTC and RUSTDOC env vars to the right executables.
-    * Git: `./y.sh prepare` uses git for applying patches and on Windows for downloading test repos.
+    * Git: Git is used for applying patches and on Windows for downloading test repos.
     * Curl and tar (non-Windows only): Used by `./y.sh prepare` to download a single commit for
       repos. Git will be used to clone the whole repo when using Windows.
     * [Hyperfine](https://github.com/sharkdp/hyperfine/): Used for benchmarking with `./y.sh bench`.

build_system/utils.rs

@@ -5,15 +5,16 @@ use std::path::{Path, PathBuf};
 use std::process::{self, Command, Stdio};
 use std::sync::atomic::{AtomicBool, Ordering};
 
-use super::path::{Dirs, RelPath};
+use crate::path::{Dirs, RelPath};
+use crate::shared_utils::rustflags_to_cmd_env;
 
 #[derive(Clone, Debug)]
 pub(crate) struct Compiler {
     pub(crate) cargo: PathBuf,
     pub(crate) rustc: PathBuf,
     pub(crate) rustdoc: PathBuf,
-    pub(crate) rustflags: String,
-    pub(crate) rustdocflags: String,
+    pub(crate) rustflags: Vec<String>,
+    pub(crate) rustdocflags: Vec<String>,
     pub(crate) triple: String,
     pub(crate) runner: Vec<String>,
 }
@@ -23,8 +24,8 @@ impl Compiler {
         match self.triple.as_str() {
             "aarch64-unknown-linux-gnu" => {
                 // We are cross-compiling for aarch64. Use the correct linker and run tests in qemu.
-                self.rustflags += " -Clinker=aarch64-linux-gnu-gcc";
-                self.rustdocflags += " -Clinker=aarch64-linux-gnu-gcc";
+                self.rustflags.push("-Clinker=aarch64-linux-gnu-gcc".to_owned());
+                self.rustdocflags.push("-Clinker=aarch64-linux-gnu-gcc".to_owned());
                 self.runner = vec![
                     "qemu-aarch64".to_owned(),
                     "-L".to_owned(),
@@ -33,8 +34,8 @@ impl Compiler {
             }
             "s390x-unknown-linux-gnu" => {
                 // We are cross-compiling for s390x. Use the correct linker and run tests in qemu.
-                self.rustflags += " -Clinker=s390x-linux-gnu-gcc";
-                self.rustdocflags += " -Clinker=s390x-linux-gnu-gcc";
+                self.rustflags.push("-Clinker=s390x-linux-gnu-gcc".to_owned());
+                self.rustdocflags.push("-Clinker=s390x-linux-gnu-gcc".to_owned());
                 self.runner = vec![
                     "qemu-s390x".to_owned(),
                     "-L".to_owned(),
@@ -100,8 +101,8 @@ impl CargoProject {
 
         cmd.env("RUSTC", &compiler.rustc);
         cmd.env("RUSTDOC", &compiler.rustdoc);
-        cmd.env("RUSTFLAGS", &compiler.rustflags);
-        cmd.env("RUSTDOCFLAGS", &compiler.rustdocflags);
+        rustflags_to_cmd_env(&mut cmd, "RUSTFLAGS", &compiler.rustflags);
+        rustflags_to_cmd_env(&mut cmd, "RUSTDOCFLAGS", &compiler.rustdocflags);
         if !compiler.runner.is_empty() {
             cmd.env(
                 format!("CARGO_TARGET_{}_RUNNER", compiler.triple.to_uppercase().replace('-', "_")),