Implement vendoring

build_system/mod.rs

@@ -29,6 +29,7 @@ macro_rules! arg_error {
 
 #[derive(PartialEq, Debug)]
 enum Command {
+    Vendor,
     Prepare,
     Build,
     Test,
@@ -59,6 +60,7 @@ pub fn main() {
 
     let mut args = env::args().skip(1);
     let command = match args.next().as_deref() {
+        Some("vendor") => Command::Vendor,
         Some("prepare") => Command::Prepare,
         Some("build") => Command::Build,
         Some("test") => Command::Test,
@@ -79,6 +81,9 @@ pub fn main() {
     while let Some(arg) = args.next().as_deref() {
         match arg {
             "--out-dir" => {
+                if command == Command::Vendor {
+                    arg_error!("vendor command doesn't accept --out-dir argument");
+                }
                 out_dir = PathBuf::from(args.next().unwrap_or_else(|| {
                     arg_error!("--out-dir requires argument");
                 }))
@@ -131,8 +136,11 @@ pub fn main() {
         std::fs::File::create(target).unwrap();
     }
 
-    if command == Command::Prepare {
-        prepare::prepare(&dirs);
+    if command == Command::Vendor {
+        prepare::prepare(&dirs, true);
+        process::exit(0);
+    } else if command == Command::Prepare {
+        prepare::prepare(&dirs, false);
         process::exit(0);
     }
 
@@ -146,7 +154,7 @@ pub fn main() {
         use_unstable_features,
     );
     match command {
-        Command::Prepare => {
+        Command::Vendor | Command::Prepare => {
             // Handled above
         }
         Command::Test => {

build_system/prepare.rs

@@ -1,32 +1,61 @@
 use std::ffi::OsStr;
 use std::fs;
 use std::path::{Path, PathBuf};
-use std::process::Command;
+use std::process::{Command, Stdio};
 
 use super::build_sysroot::{BUILD_SYSROOT, ORIG_BUILD_SYSROOT, SYSROOT_RUSTC_VERSION, SYSROOT_SRC};
 use super::path::{Dirs, RelPath};
 use super::rustc_info::{get_default_sysroot, get_rustc_version};
-use super::utils::{copy_dir_recursively, git_command, retry_spawn_and_wait, spawn_and_wait};
+use super::utils::{
+    copy_dir_recursively, git_command, remove_file_if_exists, retry_spawn_and_wait, spawn_and_wait,
+};
 
-pub(crate) fn prepare(dirs: &Dirs) {
+pub(crate) fn prepare(dirs: &Dirs, vendor: bool) {
     RelPath::DOWNLOAD.ensure_fresh(dirs);
 
-    spawn_and_wait(super::build_backend::CG_CLIF.fetch("cargo", dirs));
+    remove_file_if_exists(Path::new(".cargo/config.toml"));
+
+    let mut cargo_workspaces = vec![super::build_backend::CG_CLIF.manifest_path(dirs)];
 
     prepare_sysroot(dirs);
-    spawn_and_wait(super::build_sysroot::STANDARD_LIBRARY.fetch("cargo", dirs));
-    spawn_and_wait(super::tests::LIBCORE_TESTS.fetch("cargo", dirs));
+    cargo_workspaces.push(super::build_sysroot::STANDARD_LIBRARY.manifest_path(dirs));
+    cargo_workspaces.push(super::tests::LIBCORE_TESTS.manifest_path(dirs));
 
     super::abi_cafe::ABI_CAFE_REPO.fetch(dirs);
-    spawn_and_wait(super::abi_cafe::ABI_CAFE.fetch("cargo", dirs));
+    cargo_workspaces.push(super::abi_cafe::ABI_CAFE.manifest_path(dirs));
     super::tests::RAND_REPO.fetch(dirs);
-    spawn_and_wait(super::tests::RAND.fetch("cargo", dirs));
+    cargo_workspaces.push(super::tests::RAND.manifest_path(dirs));
     super::tests::REGEX_REPO.fetch(dirs);
-    spawn_and_wait(super::tests::REGEX.fetch("cargo", dirs));
+    cargo_workspaces.push(super::tests::REGEX.manifest_path(dirs));
     super::tests::PORTABLE_SIMD_REPO.fetch(dirs);
-    spawn_and_wait(super::tests::PORTABLE_SIMD.fetch("cargo", dirs));
+    cargo_workspaces.push(super::tests::PORTABLE_SIMD.manifest_path(dirs));
     super::bench::SIMPLE_RAYTRACER_REPO.fetch(dirs);
-    spawn_and_wait(super::bench::SIMPLE_RAYTRACER.fetch("cargo", dirs));
+    cargo_workspaces.push(super::bench::SIMPLE_RAYTRACER.manifest_path(dirs));
+
+    if vendor {
+        let mut vendor_cmd = Command::new("cargo");
+
+        vendor_cmd.arg("vendor").arg("--manifest-path").arg(&cargo_workspaces[0]);
+
+        for workspace in cargo_workspaces.iter().skip(1) {
+            vendor_cmd.arg("--sync").arg(workspace);
+        }
+
+        vendor_cmd.arg("download/vendor");
+
+        let vendor_output = vendor_cmd.stderr(Stdio::inherit()).output().unwrap();
+        assert!(vendor_output.status.success());
+        let replacement_config = String::from_utf8(vendor_output.stdout).unwrap();
+
+        std::fs::create_dir_all(".cargo").unwrap();
+        std::fs::write(".cargo/config.toml", replacement_config).unwrap();
+    } else {
+        for workspace in &cargo_workspaces {
+            let mut fetch_cmd = Command::new("cargo");
+            fetch_cmd.arg("fetch").arg("--manifest-path").arg(workspace);
+            spawn_and_wait(fetch_cmd)
+        }
+    }
 }
 
 fn prepare_sysroot(dirs: &Dirs) {

build_system/usage.txt

@@ -1,6 +1,7 @@
 The build system of cg_clif.
 
 USAGE:
+    ./y.rs vendor
     ./y.rs prepare [--out-dir DIR]
     ./y.rs build [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features]
     ./y.rs test [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features]

build_system/utils.rs

@@ -120,15 +120,6 @@ impl CargoProject {
         cmd
     }
 
-    #[must_use]
-    pub(crate) fn fetch(&self, cargo: impl AsRef<Path>, dirs: &Dirs) -> Command {
-        let mut cmd = Command::new(cargo.as_ref());
-
-        cmd.arg("fetch").arg("--manifest-path").arg(self.manifest_path(dirs));
-
-        cmd
-    }
-
     pub(crate) fn clean(&self, dirs: &Dirs) {
         let _ = fs::remove_dir_all(self.target_dir(dirs));
     }
@@ -246,6 +237,14 @@ pub(crate) fn spawn_and_wait_with_input(mut cmd: Command, input: String) -> Stri
     String::from_utf8(output.stdout).unwrap()
 }
 
+pub(crate) fn remove_file_if_exists(path: &Path) {
+    match fs::remove_file(&path) {
+        Ok(()) => {}
+        Err(err) if err.kind() == io::ErrorKind::NotFound => {}
+        Err(err) => panic!("Failed to remove {path}: {err}", path = path.display()),
+    }
+}
+
 pub(crate) fn remove_dir_if_exists(path: &Path) {
     match fs::remove_dir_all(&path) {
         Ok(()) => {}

clean_all.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -e
 
-rm -rf target/ download/ build/ dist/ y.bin y.bin.dSYM y.exe y.pdb
+rm -rf .cargo/config.toml target/ download/ build/ dist/ y.bin y.bin.dSYM y.exe y.pdb
 
 # Kept for now in case someone updates their checkout of cg_clif before running clean_all.sh
 # FIXME remove at some point in the future