Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Inline before merging cgus #112695

Merged
merged 4 commits into from
Jun 22, 2023
Merged

Inline before merging cgus #112695

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
f6cadae
Streamline some comments.
nnethercote Jun 16, 2023
6f228e3
Inline before merging CGUs.
nnethercote Jun 16, 2023
105ac1c
Merge root and inlined item placement.
nnethercote Jun 21, 2023
abde9ba
Tweak CGU size estimate code.
nnethercote Jun 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 9 additions & 11 deletions compiler/rustc_middle/src/mir/mono.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,7 @@ pub struct CodegenUnit<'tcx> {
/// as well as the crate name and disambiguator.
name: Symbol,
items: FxHashMap<MonoItem<'tcx>, (Linkage, Visibility)>,
size_estimate: Option<usize>,
size_estimate: usize,
primary: bool,
/// True if this is CGU is used to hold code coverage information for dead code,
/// false otherwise.
Expand Down Expand Up @@ -269,7 +269,7 @@ impl<'tcx> CodegenUnit<'tcx> {
CodegenUnit {
name,
items: Default::default(),
size_estimate: None,
size_estimate: 0,
primary: false,
is_code_coverage_dead_code_cgu: false,
}
Expand Down Expand Up @@ -320,23 +320,21 @@ impl<'tcx> CodegenUnit<'tcx> {
base_n::encode(hash, base_n::CASE_INSENSITIVE)
}

pub fn create_size_estimate(&mut self, tcx: TyCtxt<'tcx>) {
pub fn compute_size_estimate(&mut self, tcx: TyCtxt<'tcx>) {
// Estimate the size of a codegen unit as (approximately) the number of MIR
// statements it corresponds to.
self.size_estimate = Some(self.items.keys().map(|mi| mi.size_estimate(tcx)).sum());
self.size_estimate = self.items.keys().map(|mi| mi.size_estimate(tcx)).sum();
}

#[inline]
/// Should only be called if [`create_size_estimate`] has previously been called.
/// Should only be called if [`compute_size_estimate`] has previously been called.
///
/// [`create_size_estimate`]: Self::create_size_estimate
/// [`compute_size_estimate`]: Self::compute_size_estimate
nnethercote marked this conversation as resolved.
Show resolved Hide resolved
pub fn size_estimate(&self) -> usize {
// Items are never zero-sized, so if we have items the estimate must be
// non-zero, unless we forgot to call `compute_size_estimate` first.
assert!(self.items.is_empty() || self.size_estimate != 0);
self.size_estimate
.expect("create_size_estimate must be called before getting a size_estimate")
}

pub fn modify_size_estimate(&mut self, delta: usize) {
*self.size_estimate.as_mut().unwrap() += delta;
}

pub fn contains_item(&self, item: &MonoItem<'tcx>) -> bool {
Expand Down
137 changes: 56 additions & 81 deletions compiler/rustc_monomorphize/src/partitioning.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ struct PartitioningCx<'a, 'tcx> {
usage_map: &'a UsageMap<'tcx>,
}

struct PlacedRootMonoItems<'tcx> {
struct PlacedMonoItems<'tcx> {
/// The codegen units, sorted by name to make things deterministic.
codegen_units: Vec<CodegenUnit<'tcx>>,

Expand All @@ -150,18 +150,13 @@ where

let cx = &PartitioningCx { tcx, usage_map };

// In the first step, we place all regular monomorphizations into their
// respective 'home' codegen unit. Regular monomorphizations are all
// functions and statics defined in the local crate.
let PlacedRootMonoItems { mut codegen_units, internalization_candidates, unique_inlined_stats } = {
let _prof_timer = tcx.prof.generic_activity("cgu_partitioning_place_roots");
let mut placed = place_root_mono_items(cx, mono_items);
// Place all mono items into a codegen unit. `place_mono_items` is
// responsible for initializing the CGU size estimates.
let PlacedMonoItems { mut codegen_units, internalization_candidates, unique_inlined_stats } = {
let _prof_timer = tcx.prof.generic_activity("cgu_partitioning_place_items");
let placed = place_mono_items(cx, mono_items);

for cgu in &mut placed.codegen_units {
cgu.create_size_estimate(tcx);
}

debug_dump(tcx, "ROOTS", &placed.codegen_units, placed.unique_inlined_stats);
debug_dump(tcx, "PLACE", &placed.codegen_units, placed.unique_inlined_stats);

placed
};
Expand All @@ -175,23 +170,8 @@ where
debug_dump(tcx, "MERGE", &codegen_units, unique_inlined_stats);
}

// In the next step, we use the inlining map to determine which additional
// monomorphizations have to go into each codegen unit. These additional
// monomorphizations can be drop-glue, functions from external crates, and
// local functions the definition of which is marked with `#[inline]`.
{
let _prof_timer = tcx.prof.generic_activity("cgu_partitioning_place_inline_items");
place_inlined_mono_items(cx, &mut codegen_units);

for cgu in &mut codegen_units {
cgu.create_size_estimate(tcx);
}

debug_dump(tcx, "INLINE", &codegen_units, unique_inlined_stats);
}

// Next we try to make as many symbols "internal" as possible, so LLVM has
// more freedom to optimize.
// Make as many symbols "internal" as possible, so LLVM has more freedom to
// optimize.
if !tcx.sess.link_dead_code() {
let _prof_timer = tcx.prof.generic_activity("cgu_partitioning_internalize_symbols");
internalize_symbols(cx, &mut codegen_units, internalization_candidates);
Expand All @@ -212,10 +192,7 @@ where
codegen_units
}

fn place_root_mono_items<'tcx, I>(
cx: &PartitioningCx<'_, 'tcx>,
mono_items: I,
) -> PlacedRootMonoItems<'tcx>
fn place_mono_items<'tcx, I>(cx: &PartitioningCx<'_, 'tcx>, mono_items: I) -> PlacedMonoItems<'tcx>
where
I: Iterator<Item = MonoItem<'tcx>>,
{
Expand All @@ -236,6 +213,8 @@ where
let mut num_unique_inlined_items = 0;
let mut unique_inlined_items_size = 0;
for mono_item in mono_items {
// Handle only root items directly here. Inlined items are handled at
// the bottom of the loop based on reachability.
match mono_item.instantiation_mode(cx.tcx) {
InstantiationMode::GloballyShared { .. } => {}
InstantiationMode::LocalCopy => {
Expand All @@ -248,7 +227,7 @@ where
let characteristic_def_id = characteristic_def_id_of_mono_item(cx.tcx, mono_item);
let is_volatile = is_incremental_build && mono_item.is_generic_fn();

let codegen_unit_name = match characteristic_def_id {
let cgu_name = match characteristic_def_id {
Some(def_id) => compute_codegen_unit_name(
cx.tcx,
cgu_name_builder,
Expand All @@ -259,9 +238,7 @@ where
None => fallback_cgu_name(cgu_name_builder),
};

let codegen_unit = codegen_units
.entry(codegen_unit_name)
.or_insert_with(|| CodegenUnit::new(codegen_unit_name));
let cgu = codegen_units.entry(cgu_name).or_insert_with(|| CodegenUnit::new(cgu_name));

let mut can_be_internalized = true;
let (linkage, visibility) = mono_item_linkage_and_visibility(
Expand All @@ -274,23 +251,56 @@ where
internalization_candidates.insert(mono_item);
}

codegen_unit.items_mut().insert(mono_item, (linkage, visibility));
cgu.items_mut().insert(mono_item, (linkage, visibility));

// Get all inlined items that are reachable from `mono_item` without
// going via another root item. This includes drop-glue, functions from
// external crates, and local functions the definition of which is
// marked with `#[inline]`.
let mut reachable_inlined_items = FxHashSet::default();
get_reachable_inlined_items(cx.tcx, mono_item, cx.usage_map, &mut reachable_inlined_items);

// Add those inlined items. It's possible an inlined item is reachable
// from multiple root items within a CGU, which is fine, it just means
// the `insert` will be a no-op.
for inlined_item in reachable_inlined_items {
// This is a CGU-private copy.
cgu.items_mut().insert(inlined_item, (Linkage::Internal, Visibility::Default));
}
}

// Always ensure we have at least one CGU; otherwise, if we have a
// crate with just types (for example), we could wind up with no CGU.
if codegen_units.is_empty() {
let codegen_unit_name = fallback_cgu_name(cgu_name_builder);
codegen_units.insert(codegen_unit_name, CodegenUnit::new(codegen_unit_name));
let cgu_name = fallback_cgu_name(cgu_name_builder);
codegen_units.insert(cgu_name, CodegenUnit::new(cgu_name));
}

let mut codegen_units: Vec<_> = codegen_units.into_values().collect();
codegen_units.sort_by(|a, b| a.name().as_str().cmp(b.name().as_str()));

PlacedRootMonoItems {
for cgu in codegen_units.iter_mut() {
cgu.compute_size_estimate(cx.tcx);
}

return PlacedMonoItems {
codegen_units,
internalization_candidates,
unique_inlined_stats: (num_unique_inlined_items, unique_inlined_items_size),
};

fn get_reachable_inlined_items<'tcx>(
tcx: TyCtxt<'tcx>,
item: MonoItem<'tcx>,
usage_map: &UsageMap<'tcx>,
visited: &mut FxHashSet<MonoItem<'tcx>>,
) {
usage_map.for_each_inlined_used_item(tcx, item, |inlined_item| {
let is_new = visited.insert(inlined_item);
if is_new {
get_reachable_inlined_items(tcx, inlined_item, usage_map, visited);
}
});
}
}

Expand All @@ -314,7 +324,7 @@ fn merge_codegen_units<'tcx>(
// worse generated code. So we don't allow CGUs smaller than this (unless
// there is just one CGU, of course). Note that CGU sizes of 100,000+ are
// common in larger programs, so this isn't all that large.
const NON_INCR_MIN_CGU_SIZE: usize = 1000;
const NON_INCR_MIN_CGU_SIZE: usize = 1800;

// Repeatedly merge the two smallest codegen units as long as:
// - we have more CGUs than the upper limit, or
Expand All @@ -338,9 +348,11 @@ fn merge_codegen_units<'tcx>(
let mut smallest = codegen_units.pop().unwrap();
let second_smallest = codegen_units.last_mut().unwrap();

// Move the mono-items from `smallest` to `second_smallest`
second_smallest.modify_size_estimate(smallest.size_estimate());
// Move the items from `smallest` to `second_smallest`. Some of them
// may be duplicate inlined items, in which case the destination CGU is
// unaffected. Recalculate size estimates afterwards.
second_smallest.items_mut().extend(smallest.items_mut().drain());
second_smallest.compute_size_estimate(cx.tcx);

// Record that `second_smallest` now contains all the stuff that was
// in `smallest` before.
Expand Down Expand Up @@ -406,43 +418,6 @@ fn merge_codegen_units<'tcx>(
codegen_units.sort_by(|a, b| a.name().as_str().cmp(b.name().as_str()));
}

fn place_inlined_mono_items<'tcx>(
cx: &PartitioningCx<'_, 'tcx>,
codegen_units: &mut [CodegenUnit<'tcx>],
) {
for cgu in codegen_units.iter_mut() {
// Collect all inlined items that need to be available in this codegen unit.
let mut reachable_inlined_items = FxHashSet::default();
for root in cgu.items().keys() {
// Get all inlined items that are reachable from it without going
// via another root item.
get_reachable_inlined_items(cx.tcx, *root, cx.usage_map, &mut reachable_inlined_items);
}

// Add all monomorphizations that are not already there.
for inlined_item in reachable_inlined_items {
assert!(!cgu.items().contains_key(&inlined_item));

// This is a CGU-private copy.
cgu.items_mut().insert(inlined_item, (Linkage::Internal, Visibility::Default));
}
}

fn get_reachable_inlined_items<'tcx>(
tcx: TyCtxt<'tcx>,
item: MonoItem<'tcx>,
usage_map: &UsageMap<'tcx>,
visited: &mut FxHashSet<MonoItem<'tcx>>,
) {
usage_map.for_each_inlined_used_item(tcx, item, |inlined_item| {
let is_new = visited.insert(inlined_item);
if is_new {
get_reachable_inlined_items(tcx, inlined_item, usage_map, visited);
}
});
}
}

fn internalize_symbols<'tcx>(
cx: &PartitioningCx<'_, 'tcx>,
codegen_units: &mut [CodegenUnit<'tcx>],
Expand Down