Auto merge of #159642 - Kobzol:bootstrap-pgo-clippy, r=Mark-Simulacrum

Optimize Clippy with PGO



After rustdoc and Cargo, this seemed like an obvious target for PGO.

Perf results: https://perf.rust-lang.org/compare.html?start=390279b302ca98ae270f434100ae3730531d1246&end=caf67342a544d6eab7a40ebf0d02e802f62008a4&stat=instructions:u

- [Quick try build that doesn't optimize Clippy](#159642 (comment))
- [Full try build that optimizes Clippy](#159642 (comment))

Author: bors

bootstrap.example.toml

@@ -1028,6 +1028,11 @@
 # Instrument cargo, so that when executed, it will gather profiles
 # to this path.
 #pgo.cargo.generate = "/tmp/profiles/foo.profraw"
+# Use the following profile to PGO optimize clippy.
+#pgo.clippy.use = "/tmp/profiles/foo.profraw"
+# Instrument clippy, so that when executed, it will gather profiles
+# to this path.
+#pgo.clippy.generate = "/tmp/profiles/foo.profraw"
 # Use the following profile to PGO optimize LLVM.
 #pgo.llvm.use = "/tmp/profiles/foo.profraw"
 # Instrument LLVM, so that when executed, it will gather profiles

src/bootstrap/src/core/build_steps/dist.rs

@@ -3148,6 +3148,7 @@ impl CommandLineStep for ReproducibleArtifacts {
             &builder.config.llvm_pgo.use_profile,
             &builder.config.rustdoc_pgo.use_profile,
             &builder.config.cargo_pgo.use_profile,
+            &builder.config.clippy_pgo.use_profile,
         ];
         for profile in pgo_profiles {
             if let Some(path) = profile.as_ref() {

src/bootstrap/src/core/build_steps/tool.rs

@@ -140,6 +140,7 @@ impl Step for ToolBuild {
         let pgo_config = match self.path {
             "src/tools/rustdoc" => Some(&builder.config.rustdoc_pgo),
             "src/tools/cargo" => Some(&builder.config.cargo_pgo),
+            "src/tools/clippy" => Some(&builder.config.clippy_pgo),
             _ => None,
         };
         if let Some(pgo_config) = pgo_config {

src/bootstrap/src/core/config/config.rs

@@ -234,6 +234,7 @@ pub(crate) struct Config {
     pub rust_pgo: PgoConfig,
     pub rustdoc_pgo: PgoConfig,
     pub cargo_pgo: PgoConfig,
+    pub clippy_pgo: PgoConfig,
 
     pub stdlib_semver_baseline: Option<String>,
 
@@ -666,8 +667,13 @@ impl Config {
             libgccjit_libs_dir: gcc_libgccjit_libs_dir,
         } = toml_gcc.unwrap_or_default();
 
-        let Pgo { rustc: pgo_rustc, llvm: pgo_llvm, rustdoc: pgo_rustdoc, cargo: pgo_cargo } =
-            toml_pgo.unwrap_or_default();
+        let Pgo {
+            rustc: pgo_rustc,
+            rustdoc: pgo_rustdoc,
+            cargo: pgo_cargo,
+            clippy: pgo_clippy,
+            llvm: pgo_llvm,
+        } = toml_pgo.unwrap_or_default();
 
         // Backcompat: flags have priority over config
         if flags_rust_profile_use.is_some() || flags_rust_profile_generate.is_some() {
@@ -722,6 +728,7 @@ impl Config {
 
         let pgo_rustdoc = init_pgo(pgo_rustdoc, "rustdoc");
         let pgo_cargo = init_pgo(pgo_cargo, "cargo");
+        let pgo_clippy = init_pgo(pgo_clippy, "clippy");
 
         let bootstrap_override_lld = rust_bootstrap_override_lld.unwrap_or_default();
 
@@ -1423,6 +1430,7 @@ NOTE: Please add `--stage 2` to your command line, or if you're sure you want to
             channel,
             ci_env,
             clippy_info,
+            clippy_pgo: pgo_clippy,
             cmd: flags_cmd,
             codegen_tests: rust_codegen_tests.unwrap_or(true),
             color: flags_color,

src/bootstrap/src/core/config/toml/pgo.rs

@@ -26,6 +26,7 @@ define_config! {
         rustc: Option<PgoConfig> = "rustc",
         rustdoc: Option<PgoConfig> = "rustdoc",
         cargo: Option<PgoConfig> = "cargo",
+        clippy: Option<PgoConfig> = "clippy",
         llvm: Option<PgoConfig> = "llvm",
     }
 }

src/tools/opt-dist/src/exec.rs

@@ -7,7 +7,9 @@ use camino::{Utf8Path, Utf8PathBuf};
 use crate::environment::Environment;
 use crate::metrics::{load_metrics, record_metrics};
 use crate::timer::TimerSection;
-use crate::training::{BoltProfile, LlvmPGOProfile, RustcPGOProfile, RustdocPGOProfile};
+use crate::training::{
+    BoltProfile, ClippyPGOProfile, LlvmPGOProfile, RustcPGOProfile, RustdocPGOProfile,
+};
 use crate::utils::io::normalize_path;
 
 #[derive(Default)]
@@ -129,6 +131,11 @@ impl Bootstrap {
         self
     }
 
+    pub fn with_clippy(mut self) -> Self {
+        self.cmd = self.cmd.arg("clippy");
+        self
+    }
+
     pub fn dist(env: &Environment, dist_args: &[String]) -> Self {
         let metrics_path = env.build_root().join("metrics.json");
         let args = dist_args.iter().map(|arg| arg.as_str()).collect::<Vec<_>>();
@@ -218,6 +225,22 @@ impl Bootstrap {
         self
     }
 
+    pub fn clippy_pgo_instrument(mut self, profile_dir: &Utf8Path) -> Self {
+        self.cmd = self
+            .cmd
+            .arg("--set")
+            .arg(format!(r#"pgo.clippy.generate="{}""#, normalize_path(profile_dir).as_str()));
+        self
+    }
+
+    pub fn clippy_pgo_optimize(mut self, profile: &ClippyPGOProfile) -> Self {
+        self.cmd = self
+            .cmd
+            .arg("--set")
+            .arg(format!(r#"pgo.clippy.use="{}""#, normalize_path(&profile.0).as_str()));
+        self
+    }
+
     pub fn with_llvm_bolt_ldflags(mut self) -> Self {
         self.cmd = self.cmd.arg("--set").arg("llvm.ldflags=-Wl,-q");
         self

src/tools/opt-dist/src/main.rs

@@ -10,8 +10,8 @@ use crate::exec::{Bootstrap, cmd};
 use crate::tests::run_tests;
 use crate::timer::Timer;
 use crate::training::{
-    gather_bolt_profiles, gather_llvm_profiles, gather_rustc_profiles, gather_rustdoc_profiles,
-    llvm_benchmarks, rustc_benchmarks,
+    gather_bolt_profiles, gather_clippy_profiles, gather_llvm_profiles, gather_rustc_profiles,
+    gather_rustdoc_profiles, llvm_benchmarks, rustc_benchmarks,
 };
 use crate::utils::artifact_size::print_binary_sizes;
 use crate::utils::io::{copy_directory, reset_directory};
@@ -236,11 +236,13 @@ fn execute_pipeline(
         copy_rustc_perf(env, &rustc_perf_checkout_dir)
     })?;
 
+    let optimize_clippy = !is_fast_try_build();
+
     // Stage 1: Build PGO instrumented rustc
     // We use a normal build of LLVM, because gathering PGO profiles for LLVM and `rustc` at the
     // same time can cause issues, because the host and in-tree LLVM versions can diverge.
-    let (rustc_pgo_profile, rustdoc_pgo_profile) =
-        timer.section("Stage 1 (Rustc + rustdoc PGO)", |stage| {
+    let (rustc_pgo_profile, rustdoc_pgo_profile, clippy_pgo_profile) =
+        timer.section("Stage 1 (Rustc + rustdoc + cargo + clippy PGO)", |stage| {
             let rustc_profile_dir_root = env.artifact_dir().join("rustc-pgo");
 
             stage.section("Build PGO instrumented rustc and LLVM", |section| {
@@ -253,6 +255,9 @@ fn execute_pipeline(
                     .rustc_pgo_instrument(&rustc_profile_dir_root)
                     .cargo_pgo_instrument(&rustc_profile_dir_root)
                     .rustdoc_pgo_instrument(&rustc_profile_dir_root);
+                if optimize_clippy {
+                    builder = builder.with_clippy().clippy_pgo_instrument(&rustc_profile_dir_root);
+                }
 
                 if env.supports_shared_llvm() {
                     // This first LLVM that we build will be thrown away after this stage, and it
@@ -271,6 +276,13 @@ fn execute_pipeline(
             let rustdoc_profile = stage.section("Gather rustdoc profiles", |_| {
                 gather_rustdoc_profiles(env, &rustc_profile_dir_root)
             })?;
+            let clippy_profile = if optimize_clippy {
+                stage.section("Gather clippy profiles", |_| {
+                    Ok(Some(gather_clippy_profiles(env, &rustc_profile_dir_root)?))
+                })?
+            } else {
+                None
+            };
             print_free_disk_space()?;
 
             stage.section("Build PGO optimized rustc", |section| {
@@ -280,14 +292,17 @@ fn execute_pipeline(
                     .rustc_pgo_optimize(&rustc_profile)
                     .cargo_pgo_optimize(&rustc_profile)
                     .rustdoc_pgo_optimize(&rustdoc_profile);
+                if let Some(clippy_profile) = clippy_profile.as_ref() {
+                    cmd = cmd.with_clippy().clippy_pgo_optimize(clippy_profile);
+                }
                 if env.use_bolt() {
                     cmd = cmd.with_rustc_bolt_ldflags();
                 }
 
                 cmd.run(section)
             })?;
 
-            Ok((rustc_profile, rustdoc_profile))
+            Ok((rustc_profile, rustdoc_profile, clippy_profile))
         })?;
 
     // Stage 2: Gather LLVM PGO profiles
@@ -423,6 +438,9 @@ fn execute_pipeline(
         .rustc_pgo_optimize(&rustc_pgo_profile)
         .cargo_pgo_optimize(&rustc_pgo_profile)
         .rustdoc_pgo_optimize(&rustdoc_pgo_profile);
+    if let Some(clippy_pgo_profile) = clippy_pgo_profile {
+        dist = dist.clippy_pgo_optimize(&clippy_pgo_profile);
+    }
 
     // if LLVM is not built we'll have PGO optimized rustc
     dist = if env.supports_shared_llvm() || !env.build_llvm() {

src/tools/opt-dist/src/training.rs

@@ -115,10 +115,14 @@ pub fn rustc_benchmarks(env: &Environment) -> CmdBuilder {
     init_compiler_benchmarks(env, &["Check", "Debug", "Opt"], &["All"], RUSTC_PGO_CRATES)
 }
 
-pub fn rustdoc_benchmarks(env: &Environment) -> CmdBuilder {
+fn rustdoc_benchmarks(env: &Environment) -> CmdBuilder {
     init_compiler_benchmarks(env, &["Doc"], &["Full"], RUSTC_PGO_CRATES)
 }
 
+fn clippy_benchmarks(env: &Environment) -> CmdBuilder {
+    init_compiler_benchmarks(env, &["Clippy"], &["All"], RUSTC_PGO_CRATES)
+}
+
 pub struct LlvmPGOProfile(pub Utf8PathBuf);
 
 pub fn gather_llvm_profiles(
@@ -165,6 +169,16 @@ pub fn gather_rustdoc_profiles(
         .map(RustdocPGOProfile)
 }
 
+pub struct ClippyPGOProfile(pub Utf8PathBuf);
+
+pub fn gather_clippy_profiles(
+    env: &Environment,
+    profile_root: &Utf8Path,
+) -> anyhow::Result<ClippyPGOProfile> {
+    log::info!("Running benchmarks with PGO instrumented clippy");
+    gather_pgo_profiles(env, profile_root, "clippy", clippy_benchmarks(env)).map(ClippyPGOProfile)
+}
+
 pub fn gather_pgo_profiles(
     env: &Environment,
     profile_root: &Utf8Path,