Skip to content

Simplify LLVM bitcode linker in bootstrap #142357

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 12 additions & 6 deletions src/bootstrap/src/core/build_steps/compile.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2024,19 +2024,25 @@ impl Step for Assemble {
}
}

let maybe_install_llvm_bitcode_linker = |compiler| {
// Build llvm-bitcode-linker if it is enabled and install it into the sysroot of `compiler`
let maybe_install_llvm_bitcode_linker = |compiler: Compiler| {
if builder.config.llvm_bitcode_linker_enabled {
trace!("llvm-bitcode-linker enabled, installing");
let llvm_bitcode_linker =
builder.ensure(crate::core::build_steps::tool::LlvmBitcodeLinker {
compiler,
target: target_compiler.host,
extra_features: vec![],
target: compiler.host,
});
let tool_exe = exe("llvm-bitcode-linker", target_compiler.host);

// Copy the llvm-bitcode-linker to the self-contained binary directory
let bindir_self_contained = builder
.sysroot(compiler)
.join(format!("lib/rustlib/{}/bin/self-contained", compiler.host));
let tool_exe = exe("llvm-bitcode-linker", compiler.host);

t!(fs::create_dir_all(&bindir_self_contained));
builder.copy_link(
&llvm_bitcode_linker.tool_path,
&libdir_bin.join(tool_exe),
&bindir_self_contained.join(tool_exe),
FileType::Executable,
);
}
Expand Down
19 changes: 3 additions & 16 deletions src/bootstrap/src/core/build_steps/dist.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1555,7 +1555,7 @@ impl Step for Extended {
compiler: builder.compiler(stage, target),
backend: "cranelift".to_string(),
});
add_component!("llvm-bitcode-linker" => LlvmBitcodeLinker {compiler, target});
add_component!("llvm-bitcode-linker" => LlvmBitcodeLinker {target});

let etc = builder.src.join("src/etc/installer");

Expand Down Expand Up @@ -2323,7 +2323,6 @@ impl Step for LlvmTools {

#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)]
pub struct LlvmBitcodeLinker {
pub compiler: Compiler,
pub target: TargetSelection,
}

Expand All @@ -2338,24 +2337,12 @@ impl Step for LlvmBitcodeLinker {
}

fn make_run(run: RunConfig<'_>) {
run.builder.ensure(LlvmBitcodeLinker {
compiler: run.builder.compiler_for(
run.builder.top_stage,
run.builder.config.build,
run.target,
),
target: run.target,
});
run.builder.ensure(LlvmBitcodeLinker { target: run.target });
}

fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
let compiler = self.compiler;
let target = self.target;

builder.ensure(compile::Rustc::new(compiler, target));

let llbc_linker =
builder.ensure(tool::LlvmBitcodeLinker { compiler, target, extra_features: vec![] });
let llbc_linker = builder.ensure(tool::LlvmBitcodeLinker { target });

let self_contained_bin_dir = format!("lib/rustlib/{}/bin/self-contained", target.triple);

Expand Down
2 changes: 1 addition & 1 deletion src/bootstrap/src/core/build_steps/install.rs
Original file line number Diff line number Diff line change
Expand Up @@ -287,7 +287,7 @@ install!((self, builder, _config),
}
};
LlvmBitcodeLinker, alias = "llvm-bitcode-linker", Self::should_build(_config), only_hosts: true, {
if let Some(tarball) = builder.ensure(dist::LlvmBitcodeLinker { compiler: self.compiler, target: self.target }) {
if let Some(tarball) = builder.ensure(dist::LlvmBitcodeLinker { target: self.target }) {
install_sh(builder, "llvm-bitcode-linker", self.compiler.stage, Some(self.target), &tarball);
} else {
builder.info(
Expand Down
40 changes: 11 additions & 29 deletions src/bootstrap/src/core/build_steps/tool.rs
Original file line number Diff line number Diff line change
Expand Up @@ -983,11 +983,12 @@ impl Step for RustAnalyzerProcMacroSrv {
}
}

#[derive(Debug, Clone, Hash, PartialEq, Eq)]
/// Compile the `llvm-bitcode-linker` tool for `target`.
/// It is a compiler host tool used to link specific targets using LLVM.
/// It is used by `rustc` at runtime.
#[derive(Debug, Clone, Hash, PartialEq, Eq, PartialOrd, Ord)]
pub struct LlvmBitcodeLinker {
pub compiler: Compiler,
pub target: TargetSelection,
pub extra_features: Vec<String>,
}

impl Step for LlvmBitcodeLinker {
Expand All @@ -1002,47 +1003,28 @@ impl Step for LlvmBitcodeLinker {
}

fn make_run(run: RunConfig<'_>) {
run.builder.ensure(LlvmBitcodeLinker {
compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
extra_features: Vec::new(),
target: run.target,
});
run.builder.ensure(LlvmBitcodeLinker { target: run.target });
}

#[cfg_attr(
feature = "tracing",
instrument(level = "debug", name = "LlvmBitcodeLinker::run", skip_all)
)]
fn run(self, builder: &Builder<'_>) -> ToolBuildResult {
let tool_result = builder.ensure(ToolBuild {
compiler: self.compiler,
let compiler = builder.compiler_for_target(self.target);

builder.ensure(ToolBuild {
compiler,
target: self.target,
tool: "llvm-bitcode-linker",
mode: Mode::ToolRustc,
path: "src/tools/llvm-bitcode-linker",
source_type: SourceType::InTree,
extra_features: self.extra_features,
extra_features: vec![],
allow_features: "",
cargo_args: Vec::new(),
artifact_kind: ToolArtifactKind::Binary,
});

if tool_result.target_compiler.stage > 0 {
let bindir_self_contained = builder
.sysroot(tool_result.target_compiler)
.join(format!("lib/rustlib/{}/bin/self-contained", self.target.triple));
t!(fs::create_dir_all(&bindir_self_contained));
let bin_destination = bindir_self_contained
.join(exe("llvm-bitcode-linker", tool_result.target_compiler.host));
builder.copy_link(&tool_result.tool_path, &bin_destination, FileType::Executable);
ToolBuildResult {
tool_path: bin_destination,
build_compiler: tool_result.build_compiler,
target_compiler: tool_result.target_compiler,
}
} else {
tool_result
}
})
}
}

Expand Down
15 changes: 15 additions & 0 deletions src/bootstrap/src/core/builder/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ use tracing::instrument;

pub use self::cargo::{Cargo, cargo_profile_var};
pub use crate::Compiler;
use crate::core::build_steps::compile::Std;
use crate::core::build_steps::{
check, clean, clippy, compile, dist, doc, gcc, install, llvm, run, setup, test, tool, vendor,
};
Expand Down Expand Up @@ -1314,6 +1315,20 @@ impl<'a> Builder<'a> {
resolved_compiler
}

/// Return the lowest stage compiler that can compile code for the given `target`.
pub fn compiler_for_target(&self, target: TargetSelection) -> Compiler {
Comment on lines +1318 to +1319
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Discussion: maybe tool_compiler_for_target? I ask because this should also say something like "it must not be used to build stuff that depends on staged rustc/std", right?

Copy link
Contributor Author

@Kobzol Kobzol Jun 11, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think there's any requirement like this. It's really just "give me a compiler that can build code for target T". If stage0 could cross-compile for any target, we could use it for this always.

We actually have a bunch of situations like this in bootstrap already, but it was always done sort of implicitly. Here I wanted to make it explicit.

In other words, I think that this could be useful also for other things than just tools. But right now it's only for a single tool, ofc, so happy to rename if you want.

Copy link
Member

@jieyouxu jieyouxu Jun 11, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hm right. The reason I asked is that

this could be useful also for other things than just tools

is probably true, just that the "other things" must not use this method if they do "depend on staged compiler/std" (be it through rustc_private or directly or somehow).

For now, I think it's safer to name this as sth like tool_compiler_for_target to make it clear that you should not use it for sth beyond its purpose (because we also have Builder::{compiler,compiler_for}), and then consider its naming if we do use it for not-just-tools, but it's not really a blocking concern.

// If we're not cross-compiling, we can always use the stage0 compiler
if self.config.build == target {
self.compiler(0, target)
} else {
// Otherwise, we have to build a stage 1 compiler that can compile code for `target`.
let compiler = self.compiler(1, self.config.build);
// FIXME(kobzol): get rid of this nonsense and create something like `RustcWithStdForTarget`
self.ensure(Std::new(compiler, target));
compiler
}
}

pub fn sysroot(&self, compiler: Compiler) -> PathBuf {
self.ensure(compile::Sysroot::new(compiler))
}
Expand Down
27 changes: 27 additions & 0 deletions src/bootstrap/src/core/builder/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ use llvm::prebuilt_llvm_config;
use super::*;
use crate::Flags;
use crate::core::build_steps::doc::DocumentationFormat;
use crate::core::build_steps::tool::LlvmBitcodeLinker;
use crate::core::config::Config;
use crate::utils::tests::git::{GitCtx, git_test};

Expand Down Expand Up @@ -1233,3 +1234,29 @@ fn any_debug() {
// Downcasting to the underlying type should succeed.
assert_eq!(x.downcast_ref::<MyStruct>(), Some(&MyStruct { x: 7 }));
}

/// Check that during a non-cross-compiling stage 2 build, we only compile rustc host tools
/// (such as llvm-bitcode-linker) only once.
#[test]
fn llvm_bitcode_linker_compile_once() {
Comment on lines +1238 to +1241
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Question: should we have another test that does check the cross-compiling case, that we do build a stage 1 (rustc, std) pair that can produce artifacts for target, and that llvm-bitcode-linker is produced by that pair for the target?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure, happy to add it. Although from my local experiments, I'm almost sure that bootstrap is actually broken for these situations somehow. Like, ./x build --host i686-unknown-linux-gnu, which is like the simplest cross-compilation scenario on x64 Linux, fails for me on master.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

And you would be right 🎡

let mut cache = run_build(
&[],
configure_with_args(
&[
"build".to_string(),
"--stage".to_string(),
"2".to_string(),
"--set".to_string(),
"rust.llvm-bitcode-linker=true".to_string(),
],
&[TEST_TRIPLE_1],
&[TEST_TRIPLE_2],
),
);

// Check that llvm-bitcode-linker was built only once, and for host, not target
assert_eq!(
first(cache.all::<LlvmBitcodeLinker>()),
&[LlvmBitcodeLinker { target: TargetSelection::from_user(TEST_TRIPLE_1) }]
);
}
Loading