feat(deps): pick and choose parts of brew bundle exec

cargo-dist/src/errors.rs

@@ -259,6 +259,15 @@ pub enum DistError {
     /// random camino conversion error
     #[error(transparent)]
     FromPathBufError(#[from] camino::FromPathBufError),
+
+    /// Error parsing a string containing an environment variable
+    /// in VAR=value syntax
+    #[error("Unable to parse environment variable as a key/value pair: {line}")]
+    #[diagnostic(help("This should be impossible, you did nothing wrong, please file an issue!"))]
+    EnvParseError {
+        /// The line of text that couldn't be parsed
+        line: String,
+    },
 }
 
 impl From<minijinja::Error> for DistError {

cargo-dist/src/lib.rs

@@ -12,6 +12,7 @@
 
 use std::{
     collections::{BTreeMap, HashMap},
+    env,
     fs::{self, File},
     io::{Cursor, Read},
     process::Command,
@@ -395,13 +396,144 @@ fn write_checksum(checksum: &str, src_path: &Utf8Path, dest_path: &Utf8Path) ->
     Ok(())
 }
 
+// Takes a string in KEY=value environment variable format and
+// parses it into a BTreeMap. The string syntax is sh-compatible, and also the
+// format returned by `env`.
+// Note that we trust the parsed string to contain a given key only once;
+// if specified more than once, only the final occurrence will be included.
+fn parse_env(env_string: &str) -> DistResult<BTreeMap<&str, &str>> {
+    let mut parsed = BTreeMap::new();
+    for line in env_string.trim_end().split('\n') {
+        let Some((key, value)) = line.split_once('=') else {
+            return Err(DistError::EnvParseError {
+                line: line.to_owned(),
+            });
+        };
+        parsed.insert(key, value);
+    }
+
+    Ok(parsed)
+}
+
+/// Given the environment captured from `brew bundle exec -- env`, returns
+/// a list of all dependencies from that environment and the opt prefixes
+/// to those packages.
+fn formulas_from_env(environment: &BTreeMap<&str, &str>) -> Vec<(String, String)> {
+    let mut packages = vec![];
+
+    // Set by Homebrew/brew bundle - a comma-separated list of all
+    // dependencies in the recursive tree calculated from the dependencies
+    // in the Brewfile.
+    if let Some(formulastring) = environment.get("HOMEBREW_DEPENDENCIES") {
+        // Set by Homebrew/brew bundle - the path to Homebrew's "opt"
+        // directory, which is where links to the private cellar of every
+        // installed package lives.
+        // Usually /opt/homebrew/opt or /usr/local/opt.
+        if let Some(opt_prefix) = environment.get("HOMEBREW_OPT") {
+            for dep in formulastring.split(',') {
+                // Unwrap here is safe because `split` will always return
+                // a collection of at least one item.
+                let short_name = dep.split('/').last().unwrap();
+                let pkg_opt = format!("{opt_prefix}/{short_name}");
+                packages.push((dep.to_owned(), pkg_opt));
+            }
+        }
+    }
+
+    packages
+}
+
+/// Takes a BTreeMap of key/value environment variables produced by
+/// `brew bundle exec` and decides which ones we want to keep for our own builds.
+/// Returns a Vec containing (KEY, value) tuples.
+fn select_brew_env(environment: &BTreeMap<&str, &str>) -> Vec<(String, String)> {
+    let mut desired_env = vec![];
+
+    // Several of Homebrew's environment variables are safe for us to use
+    // unconditionally, so pick those in their entirety.
+    if let Some(value) = environment.get("PKG_CONFIG_PATH") {
+        desired_env.push(("PKG_CONFIG_PATH".to_owned(), value.to_string()))
+    }
+    if let Some(value) = environment.get("PKG_CONFIG_LIBDIR") {
+        desired_env.push(("PKG_CONFIG_LIBDIR".to_owned(), value.to_string()))
+    }
+    if let Some(value) = environment.get("CMAKE_INCLUDE_PATH") {
+        desired_env.push(("CMAKE_INCLUDE_PATH".to_owned(), value.to_string()))
+    }
+    if let Some(value) = environment.get("CMAKE_LIBRARY_PATH") {
+        desired_env.push(("CMAKE_LIBRARY_PATH".to_owned(), value.to_string()))
+    }
+    let mut paths = vec![];
+
+    // For each listed dependency, add it to the PATH
+    for (_, pkg_opt) in formulas_from_env(environment) {
+        // Not every package will have a /bin or /sbin directory,
+        // but it's safe to add both to the PATH just in case.
+        paths.push(format!("{pkg_opt}/bin"));
+        paths.push(format!("{pkg_opt}/sbin"));
+    }
+
+    if !paths.is_empty() {
+        let our_path = env!("PATH");
+        let desired_path = format!("{our_path}:{}", paths.join(":"));
+
+        desired_env.insert(0, ("PATH".to_owned(), desired_path));
+    }
+
+    desired_env
+}
+
+/// Similar to the above, we read Homebrew's recursive dependency tree and
+/// then append link flags to cargo-dist's rustflags.
+/// These ensure that Rust can find C libraries that may exist within
+/// each package's prefix.
+fn determine_brew_rustflags(base_rustflags: &str, environment: &BTreeMap<&str, &str>) -> String {
+    let mut rustflags = base_rustflags.to_owned();
+    // For each listed dependency, add it to CFLAGS/LDFLAGS
+    for (_, pkg_opt) in formulas_from_env(environment) {
+        // Note that this path might not actually exist; not every
+        // package contains libraries. However, it's safe to
+        // append this flag anyway; Rust passes it on to the
+        // compiler/linker, which tolerate missing directories
+        // just fine.
+        rustflags = format!("{rustflags} -L{pkg_opt}/lib");
+    }
+
+    rustflags
+}
+
 /// Build a cargo target
 fn build_cargo_target(dist_graph: &DistGraph, target: &CargoBuildStep) -> Result<()> {
     eprint!(
         "building cargo target ({}/{}",
         target.target_triple, target.profile
     );
 
+    let mut rustflags = target.rustflags.clone();
+    let mut desired_extra_env = vec![];
+    let skip_brewfile = env::var("DO_NOT_USE_BREWFILE").is_ok();
+    if let Some(brew) = &dist_graph.tools.brew {
+        if Utf8Path::new("Brewfile").exists() && !skip_brewfile {
+            // Uses `brew bundle exec` to just print its own environment,
+            // allowing us to capture what it generated and decide what
+            // to do with it.
+            let result = Command::new(&brew.cmd)
+                .arg("bundle")
+                .arg("exec")
+                .arg("--")
+                .arg("/usr/bin/env")
+                .output()
+                .into_diagnostic()
+                .wrap_err_with(|| "failed to exec brew bundle exec".to_string())?;
+
+            let env_output = String::from_utf8_lossy(&result.stdout).to_string();
+
+            let brew_env = parse_env(&env_output)?;
+            desired_extra_env = select_brew_env(&brew_env);
+            rustflags = determine_brew_rustflags(&rustflags, &brew_env);
+        }
+    }
+
     let mut command = Command::new(&dist_graph.tools.cargo.cmd);
     command
         .arg("build")
@@ -410,7 +542,7 @@ fn build_cargo_target(dist_graph: &DistGraph, target: &CargoBuildStep) -> Result
         .arg("--message-format=json-render-diagnostics")
         .arg("--target")
         .arg(&target.target_triple)
-        .env("RUSTFLAGS", &target.rustflags)
+        .env("RUSTFLAGS", &rustflags)
         .stdout(std::process::Stdio::piped());
     if !target.features.default_features {
         command.arg("--no-default-features");
@@ -438,6 +570,9 @@ fn build_cargo_target(dist_graph: &DistGraph, target: &CargoBuildStep) -> Result
             eprintln!(" --package={})", package);
         }
     }
+    // If we generated any extra environment variables to
+    // inject into the environment, apply them now.
+    command.envs(desired_extra_env);
     info!("exec: {:?}", command);
     let mut task = command
         .spawn()

cargo-dist/src/tasks.rs

@@ -218,6 +218,8 @@ pub struct Tools {
     pub cargo: CargoInfo,
     /// rustup, useful for getting specific toolchains
     pub rustup: Option<Tool>,
+    /// homebrew, only available on macOS
+    pub brew: Option<Tool>,
 }
 
 /// Info about the cargo toolchain we're using
@@ -2568,12 +2570,13 @@ fn tool_info() -> Result<Tools> {
     let cargo = get_host_target(cargo_cmd)?;
     Ok(Tools {
         cargo,
-        rustup: find_tool("rustup"),
+        rustup: find_tool("rustup", "-V"),
+        brew: find_tool("brew", "--version"),
     })
 }
 
-fn find_tool(name: &str) -> Option<Tool> {
-    let output = Command::new(name).arg("-V").output().ok()?;
+fn find_tool(name: &str, test_flag: &str) -> Option<Tool> {
+    let output = Command::new(name).arg(test_flag).output().ok()?;
     let string_output = String::from_utf8(output.stdout).ok()?;
     let version = string_output.lines().next()?;
     Some(Tool {

cargo-dist/src/tests/mock.rs

@@ -45,6 +45,7 @@ pub fn mock_tools() -> Tools {
             host_target: "x86_64-unknown-linux-gnu".to_owned(),
         },
         rustup: None,
+        brew: None,
     }
 }
 

cargo-dist/templates/ci/github_ci.yml.j2

@@ -120,17 +120,10 @@ jobs:
         run: |
           ${{ matrix.packages_install }}
       - name: Build artifacts
-        if: ${{ hashFiles('Brewfile') == '' }}
         run: |
           # Actually do builds and make zips and whatnot
           cargo dist build ${{ needs.plan.outputs.tag-flag }} --print=linkage --output-format=json ${{ matrix.dist_args }} > dist-manifest.json
           echo "cargo dist ran successfully"
-      - name: Build artifacts (using Brewfile)
-        if: ${{ hashFiles('Brewfile') != '' }}
-        run: |
-          # Actually do builds and make zips and whatnot
-          brew bundle exec -- cargo dist build ${{ needs.plan.outputs.tag-flag }} --print=linkage --output-format=json ${{ matrix.dist_args }} > dist-manifest.json
-          echo "cargo dist ran successfully"
       - id: cargo-dist
         name: Post-build
         # We force bash here just because github makes it really hard to get values up

cargo-dist/tests/snapshots/akaikatana_basic.snap

@@ -1410,17 +1410,10 @@ jobs:
         run: |
           ${{ matrix.packages_install }}
       - name: Build artifacts
-        if: ${{ hashFiles('Brewfile') == '' }}
         run: |
           # Actually do builds and make zips and whatnot
           cargo dist build ${{ needs.plan.outputs.tag-flag }} --print=linkage --output-format=json ${{ matrix.dist_args }} > dist-manifest.json
           echo "cargo dist ran successfully"
-      - name: Build artifacts (using Brewfile)
-        if: ${{ hashFiles('Brewfile') != '' }}
-        run: |
-          # Actually do builds and make zips and whatnot
-          brew bundle exec -- cargo dist build ${{ needs.plan.outputs.tag-flag }} --print=linkage --output-format=json ${{ matrix.dist_args }} > dist-manifest.json
-          echo "cargo dist ran successfully"
       - id: cargo-dist
         name: Post-build
         # We force bash here just because github makes it really hard to get values up

cargo-dist/tests/snapshots/akaikatana_musl.snap

@@ -1094,17 +1094,10 @@ jobs:
         run: |
           ${{ matrix.packages_install }}
       - name: Build artifacts
-        if: ${{ hashFiles('Brewfile') == '' }}
         run: |
           # Actually do builds and make zips and whatnot
           cargo dist build ${{ needs.plan.outputs.tag-flag }} --print=linkage --output-format=json ${{ matrix.dist_args }} > dist-manifest.json
           echo "cargo dist ran successfully"
-      - name: Build artifacts (using Brewfile)
-        if: ${{ hashFiles('Brewfile') != '' }}
-        run: |
-          # Actually do builds and make zips and whatnot
-          brew bundle exec -- cargo dist build ${{ needs.plan.outputs.tag-flag }} --print=linkage --output-format=json ${{ matrix.dist_args }} > dist-manifest.json
-          echo "cargo dist ran successfully"
       - id: cargo-dist
         name: Post-build
         # We force bash here just because github makes it really hard to get values up

cargo-dist/tests/snapshots/akaikatana_repo_with_dot_git.snap

@@ -1410,17 +1410,10 @@ jobs:
         run: |
           ${{ matrix.packages_install }}
       - name: Build artifacts
-        if: ${{ hashFiles('Brewfile') == '' }}
         run: |
           # Actually do builds and make zips and whatnot
           cargo dist build ${{ needs.plan.outputs.tag-flag }} --print=linkage --output-format=json ${{ matrix.dist_args }} > dist-manifest.json
           echo "cargo dist ran successfully"
-      - name: Build artifacts (using Brewfile)
-        if: ${{ hashFiles('Brewfile') != '' }}
-        run: |
-          # Actually do builds and make zips and whatnot
-          brew bundle exec -- cargo dist build ${{ needs.plan.outputs.tag-flag }} --print=linkage --output-format=json ${{ matrix.dist_args }} > dist-manifest.json
-          echo "cargo dist ran successfully"
       - id: cargo-dist
         name: Post-build
         # We force bash here just because github makes it really hard to get values up

cargo-dist/tests/snapshots/axolotlsay_basic.snap

@@ -2309,17 +2309,10 @@ jobs:
         run: |
           ${{ matrix.packages_install }}
       - name: Build artifacts
-        if: ${{ hashFiles('Brewfile') == '' }}
         run: |
           # Actually do builds and make zips and whatnot
           cargo dist build ${{ needs.plan.outputs.tag-flag }} --print=linkage --output-format=json ${{ matrix.dist_args }} > dist-manifest.json
           echo "cargo dist ran successfully"
-      - name: Build artifacts (using Brewfile)
-        if: ${{ hashFiles('Brewfile') != '' }}
-        run: |
-          # Actually do builds and make zips and whatnot
-          brew bundle exec -- cargo dist build ${{ needs.plan.outputs.tag-flag }} --print=linkage --output-format=json ${{ matrix.dist_args }} > dist-manifest.json
-          echo "cargo dist ran successfully"
       - id: cargo-dist
         name: Post-build
         # We force bash here just because github makes it really hard to get values up

cargo-dist/tests/snapshots/axolotlsay_edit_existing.snap

@@ -2284,17 +2284,10 @@ jobs:
         run: |
           ${{ matrix.packages_install }}
       - name: Build artifacts
-        if: ${{ hashFiles('Brewfile') == '' }}
         run: |
           # Actually do builds and make zips and whatnot
           cargo dist build ${{ needs.plan.outputs.tag-flag }} --print=linkage --output-format=json ${{ matrix.dist_args }} > dist-manifest.json
           echo "cargo dist ran successfully"
-      - name: Build artifacts (using Brewfile)
-        if: ${{ hashFiles('Brewfile') != '' }}
-        run: |
-          # Actually do builds and make zips and whatnot
-          brew bundle exec -- cargo dist build ${{ needs.plan.outputs.tag-flag }} --print=linkage --output-format=json ${{ matrix.dist_args }} > dist-manifest.json
-          echo "cargo dist ran successfully"
       - id: cargo-dist
         name: Post-build
         # We force bash here just because github makes it really hard to get values up

cargo-dist/tests/snapshots/axolotlsay_musl.snap

@@ -1972,17 +1972,10 @@ jobs:
         run: |
           ${{ matrix.packages_install }}
       - name: Build artifacts
-        if: ${{ hashFiles('Brewfile') == '' }}
         run: |
           # Actually do builds and make zips and whatnot
           cargo dist build ${{ needs.plan.outputs.tag-flag }} --print=linkage --output-format=json ${{ matrix.dist_args }} > dist-manifest.json
           echo "cargo dist ran successfully"
-      - name: Build artifacts (using Brewfile)
-        if: ${{ hashFiles('Brewfile') != '' }}
-        run: |
-          # Actually do builds and make zips and whatnot
-          brew bundle exec -- cargo dist build ${{ needs.plan.outputs.tag-flag }} --print=linkage --output-format=json ${{ matrix.dist_args }} > dist-manifest.json
-          echo "cargo dist ran successfully"
       - id: cargo-dist
         name: Post-build
         # We force bash here just because github makes it really hard to get values up

cargo-dist/tests/snapshots/axolotlsay_no_homebrew_publish.snap

@@ -2284,17 +2284,10 @@ jobs:
         run: |
           ${{ matrix.packages_install }}
       - name: Build artifacts
-        if: ${{ hashFiles('Brewfile') == '' }}
         run: |
           # Actually do builds and make zips and whatnot
           cargo dist build ${{ needs.plan.outputs.tag-flag }} --print=linkage --output-format=json ${{ matrix.dist_args }} > dist-manifest.json
           echo "cargo dist ran successfully"
-      - name: Build artifacts (using Brewfile)
-        if: ${{ hashFiles('Brewfile') != '' }}
-        run: |
-          # Actually do builds and make zips and whatnot
-          brew bundle exec -- cargo dist build ${{ needs.plan.outputs.tag-flag }} --print=linkage --output-format=json ${{ matrix.dist_args }} > dist-manifest.json
-          echo "cargo dist ran successfully"
       - id: cargo-dist
         name: Post-build
         # We force bash here just because github makes it really hard to get values up

cargo-dist/tests/snapshots/axolotlsay_ssldotcom_windows_sign.snap

@@ -1398,17 +1398,10 @@ jobs:
         run: |
           ${{ matrix.packages_install }}
       - name: Build artifacts
-        if: ${{ hashFiles('Brewfile') == '' }}
         run: |
           # Actually do builds and make zips and whatnot
           cargo dist build ${{ needs.plan.outputs.tag-flag }} --print=linkage --output-format=json ${{ matrix.dist_args }} > dist-manifest.json
           echo "cargo dist ran successfully"
-      - name: Build artifacts (using Brewfile)
-        if: ${{ hashFiles('Brewfile') != '' }}
-        run: |
-          # Actually do builds and make zips and whatnot
-          brew bundle exec -- cargo dist build ${{ needs.plan.outputs.tag-flag }} --print=linkage --output-format=json ${{ matrix.dist_args }} > dist-manifest.json
-          echo "cargo dist ran successfully"
       - id: cargo-dist
         name: Post-build
         # We force bash here just because github makes it really hard to get values up