Skip to content

Commit

Permalink
Rollup merge of rust-lang#37220 - Mark-Simulacrum:arena-alloc-slice, …
Browse files Browse the repository at this point in the history
…r=eddyb

alloc_slice in TypedArena

Added `TypedArena::alloc_slice`, and moved from using `TypedArena<Vec<T>>` to `TypedArena<T>`.

`TypedArena::alloc_slice` is implemented by copying the slices elements into the typed arena, requiring that `T: Copy` when using it. We allocate a new chunk when there's insufficient space remaining in the previous chunk, and we cannot resize the old chunk in place. This is non-optimal, since we may waste allocated space when allocating (especially longer) slices, but is considered good enough for the time being.

This change also reduces heap fragmentation, since the arena now directly stores objects instead of storing the Vec's length and pointer to its contents.

Performance:
```
futures-rs-test  5.048s vs  5.061s --> 0.997x faster (variance: 1.028x, 1.020x)
helloworld       0.284s vs  0.295s --> 0.963x faster (variance: 1.207x, 1.189x)
html5ever-2016-  8.396s vs  8.208s --> 1.023x faster (variance: 1.019x, 1.036x)
hyper.0.5.0      5.768s vs  5.797s --> 0.995x faster (variance: 1.027x, 1.028x)
inflate-0.1.0    5.213s vs  5.069s --> 1.028x faster (variance: 1.008x, 1.022x)
issue-32062-equ  0.428s vs  0.467s --> 0.916x faster (variance: 1.188x, 1.015x)
issue-32278-big  1.949s vs  2.010s --> 0.970x faster (variance: 1.112x, 1.049x)
jld-day15-parse  1.795s vs  1.877s --> 0.956x faster (variance: 1.037x, 1.015x)
piston-image-0. 13.554s vs 13.522s --> 1.002x faster (variance: 1.019x, 1.020x)
rust-encoding-0  2.489s vs  2.465s --> 1.010x faster (variance: 1.047x, 1.086x)
syntex-0.42.2   34.646s vs 34.593s --> 1.002x faster (variance: 1.007x, 1.005x)
syntex-0.42.2-i 17.181s vs 17.163s --> 1.001x faster (variance: 1.004x, 1.004x)
```

r? @eddyb
  • Loading branch information
eddyb authored Oct 19, 2016
2 parents ba69fd7 + 7a38599 commit ce9e95a
Show file tree
Hide file tree
Showing 19 changed files with 94 additions and 57 deletions.
45 changes: 39 additions & 6 deletions src/libarena/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ use std::intrinsics;
use std::marker::{PhantomData, Send};
use std::mem;
use std::ptr;
use std::slice;

use alloc::heap;
use alloc::raw_vec::RawVec;
Expand Down Expand Up @@ -133,7 +134,7 @@ impl<T> TypedArena<T> {
#[inline]
pub fn alloc(&self, object: T) -> &mut T {
if self.ptr == self.end {
self.grow()
self.grow(1)
}

unsafe {
Expand All @@ -154,24 +155,56 @@ impl<T> TypedArena<T> {
}
}

/// Allocates a slice of objects that are copy into the `TypedArena`, returning a mutable
/// reference to it. Will panic if passed a zero-sized types.
#[inline]
pub fn alloc_slice(&self, slice: &[T]) -> &mut [T]
where T: Copy {
assert!(mem::size_of::<T>() != 0);
if slice.len() == 0 {
return unsafe { slice::from_raw_parts_mut(heap::EMPTY as *mut T, 0) };
}

let available_capacity_bytes = self.end.get() as usize - self.ptr.get() as usize;
let at_least_bytes = slice.len() * mem::size_of::<T>();
if available_capacity_bytes < at_least_bytes {
self.grow(slice.len());
}

unsafe {
let start_ptr = self.ptr.get();
let arena_slice = slice::from_raw_parts_mut(start_ptr, slice.len());
self.ptr.set(start_ptr.offset(arena_slice.len() as isize));
arena_slice.copy_from_slice(slice);
arena_slice
}
}

/// Grows the arena.
#[inline(never)]
#[cold]
fn grow(&self) {
fn grow(&self, n: usize) {
unsafe {
let mut chunks = self.chunks.borrow_mut();
let (chunk, new_capacity);
let (chunk, mut new_capacity);
if let Some(last_chunk) = chunks.last_mut() {
if last_chunk.storage.double_in_place() {
let used_bytes = self.ptr.get() as usize - last_chunk.start() as usize;
let currently_used_cap = used_bytes / mem::size_of::<T>();
if last_chunk.storage.reserve_in_place(currently_used_cap, n) {
self.end.set(last_chunk.end());
return;
} else {
let prev_capacity = last_chunk.storage.cap();
new_capacity = prev_capacity.checked_mul(2).unwrap();
loop {
new_capacity = prev_capacity.checked_mul(2).unwrap();
if new_capacity >= currently_used_cap + n {
break;
}
}
}
} else {
let elem_size = cmp::max(1, mem::size_of::<T>());
new_capacity = cmp::max(1, PAGE / elem_size);
new_capacity = cmp::max(n, PAGE / elem_size);
}
chunk = TypedArenaChunk::<T>::new(new_capacity);
self.ptr.set(chunk.start());
Expand Down
4 changes: 2 additions & 2 deletions src/librustc/mir/tcx.rs
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@ impl<'tcx> Rvalue<'tcx> {
let lhs_ty = lhs.ty(mir, tcx);
let rhs_ty = rhs.ty(mir, tcx);
let ty = op.ty(tcx, lhs_ty, rhs_ty);
let ty = tcx.mk_tup(vec![ty, tcx.types.bool]);
let ty = tcx.mk_tup(&[ty, tcx.types.bool]);
Some(ty)
}
&Rvalue::UnaryOp(_, ref operand) => {
Expand All @@ -184,7 +184,7 @@ impl<'tcx> Rvalue<'tcx> {
}
AggregateKind::Tuple => {
Some(tcx.mk_tup(
ops.iter().map(|op| op.ty(mir, tcx)).collect()
&ops.iter().map(|op| op.ty(mir, tcx)).collect::<Vec<_>>()
))
}
AggregateKind::Adt(def, _, substs, _) => {
Expand Down
2 changes: 1 addition & 1 deletion src/librustc/traits/util.rs
Original file line number Diff line number Diff line change
Expand Up @@ -486,7 +486,7 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> {
{
let arguments_tuple = match tuple_arguments {
TupleArgumentsFlag::No => sig.0.inputs[0],
TupleArgumentsFlag::Yes => self.mk_tup(sig.0.inputs.to_vec()),
TupleArgumentsFlag::Yes => self.mk_tup(&sig.0.inputs),
};
let trait_ref = ty::TraitRef {
def_id: fn_trait_def_id,
Expand Down
38 changes: 21 additions & 17 deletions src/librustc/ty/context.rs
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,8 @@ use hir;
pub struct CtxtArenas<'tcx> {
// internings
type_: TypedArena<TyS<'tcx>>,
type_list: TypedArena<Vec<Ty<'tcx>>>,
substs: TypedArena<Vec<Kind<'tcx>>>,
type_list: TypedArena<Ty<'tcx>>,
substs: TypedArena<Kind<'tcx>>,
bare_fn: TypedArena<BareFnTy<'tcx>>,
region: TypedArena<Region>,
stability: TypedArena<attr::Stability>,
Expand Down Expand Up @@ -1117,6 +1117,7 @@ impl<'tcx> Borrow<Region> for Interned<'tcx, Region> {

macro_rules! intern_method {
($lt_tcx:tt, $name:ident: $method:ident($alloc:ty,
$alloc_method:ident,
$alloc_to_key:expr,
$alloc_to_ret:expr,
$needs_infer:expr) -> $ty:ty) => {
Expand All @@ -1142,7 +1143,8 @@ macro_rules! intern_method {
let v = unsafe {
mem::transmute(v)
};
let i = ($alloc_to_ret)(self.global_interners.arenas.$name.alloc(v));
let i = ($alloc_to_ret)(self.global_interners.arenas.$name
.$alloc_method(v));
self.global_interners.$name.borrow_mut().insert(Interned(i));
return i;
}
Expand All @@ -1156,7 +1158,7 @@ macro_rules! intern_method {
}
}

let i = ($alloc_to_ret)(self.interners.arenas.$name.alloc(v));
let i = ($alloc_to_ret)(self.interners.arenas.$name.$alloc_method(v));
self.interners.$name.borrow_mut().insert(Interned(i));
i
}
Expand All @@ -1180,7 +1182,7 @@ macro_rules! direct_interners {
}
}

intern_method!($lt_tcx, $name: $method($ty, |x| x, |x| x, $needs_infer) -> $ty);)+
intern_method!($lt_tcx, $name: $method($ty, alloc, |x| x, |x| x, $needs_infer) -> $ty);)+
}
}

Expand All @@ -1200,16 +1202,18 @@ direct_interners!('tcx,
}) -> Region
);

intern_method!('tcx,
type_list: mk_type_list(Vec<Ty<'tcx>>, Deref::deref, |xs: &[Ty]| -> &Slice<Ty> {
unsafe { mem::transmute(xs) }
}, keep_local) -> Slice<Ty<'tcx>>
);
macro_rules! slice_interners {
($($field:ident: $method:ident($ty:ident)),+) => (
$(intern_method!('tcx, $field: $method(&[$ty<'tcx>], alloc_slice, Deref::deref,
|xs: &[$ty]| -> &Slice<$ty> {
unsafe { mem::transmute(xs) }
}, |xs: &[$ty]| xs.iter().any(keep_local)) -> Slice<$ty<'tcx>>);)+
)
}

intern_method!('tcx,
substs: mk_substs(Vec<Kind<'tcx>>, Deref::deref, |xs: &[Kind]| -> &Slice<Kind> {
unsafe { mem::transmute(xs) }
}, keep_local) -> Slice<Kind<'tcx>>
slice_interners!(
type_list: mk_type_list(Ty),
substs: mk_substs(Kind)
);

impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> {
Expand Down Expand Up @@ -1314,12 +1318,12 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> {
self.mk_ty(TySlice(ty))
}

pub fn mk_tup(self, ts: Vec<Ty<'tcx>>) -> Ty<'tcx> {
pub fn mk_tup(self, ts: &[Ty<'tcx>]) -> Ty<'tcx> {
self.mk_ty(TyTuple(self.mk_type_list(ts)))
}

pub fn mk_nil(self) -> Ty<'tcx> {
self.mk_tup(Vec::new())
self.mk_tup(&[])
}

pub fn mk_diverging_default(self) -> Ty<'tcx> {
Expand Down Expand Up @@ -1361,7 +1365,7 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> {
pub fn mk_closure(self,
closure_id: DefId,
substs: &'tcx Substs<'tcx>,
tys: Vec<Ty<'tcx>>)
tys: &[Ty<'tcx>])
-> Ty<'tcx> {
self.mk_closure_from_closure_substs(closure_id, ClosureSubsts {
func_substs: substs,
Expand Down
2 changes: 1 addition & 1 deletion src/librustc/ty/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1797,7 +1797,7 @@ impl<'a, 'tcx> AdtDefData<'tcx, 'tcx> {
_ if tys.references_error() => tcx.types.err,
0 => tcx.types.bool,
1 => tys[0],
_ => tcx.mk_tup(tys)
_ => tcx.mk_tup(&tys)
};

match self.sized_constraint.get(dep_node()) {
Expand Down
6 changes: 3 additions & 3 deletions src/librustc/ty/relate.rs
Original file line number Diff line number Diff line change
Expand Up @@ -491,8 +491,8 @@ pub fn super_relate_tys<'a, 'gcx, 'tcx, R>(relation: &mut R,
if as_.len() == bs.len() {
let ts = as_.iter().zip(bs)
.map(|(a, b)| relation.relate(a, b))
.collect::<Result<_, _>>()?;
Ok(tcx.mk_tup(ts))
.collect::<Result<Vec<_>, _>>()?;
Ok(tcx.mk_tup(&ts))
} else if !(as_.is_empty() || bs.is_empty()) {
Err(TypeError::TupleSize(
expected_found(relation, &as_.len(), &bs.len())))
Expand Down Expand Up @@ -547,7 +547,7 @@ impl<'tcx> Relate<'tcx> for ty::ClosureSubsts<'tcx> {
let upvar_tys = relation.relate_zip(&a.upvar_tys, &b.upvar_tys)?;
Ok(ty::ClosureSubsts {
func_substs: substs,
upvar_tys: relation.tcx().mk_type_list(upvar_tys)
upvar_tys: relation.tcx().mk_type_list(&upvar_tys)
})
}
}
Expand Down
4 changes: 2 additions & 2 deletions src/librustc/ty/structural_impls.rs
Original file line number Diff line number Diff line change
Expand Up @@ -448,8 +448,8 @@ impl<'tcx> TypeFoldable<'tcx> for ty::TraitObject<'tcx> {

impl<'tcx> TypeFoldable<'tcx> for &'tcx ty::Slice<Ty<'tcx>> {
fn super_fold_with<'gcx: 'tcx, F: TypeFolder<'gcx, 'tcx>>(&self, folder: &mut F) -> Self {
let tys = self.iter().map(|t| t.fold_with(folder)).collect();
folder.tcx().mk_type_list(tys)
let tys = self.iter().map(|t| t.fold_with(folder)).collect::<Vec<_>>();
folder.tcx().mk_type_list(&tys)
}

fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool {
Expand Down
6 changes: 3 additions & 3 deletions src/librustc/ty/subst.rs
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,7 @@ impl<'a, 'gcx, 'tcx> Substs<'tcx> {
pub fn new<I>(tcx: TyCtxt<'a, 'gcx, 'tcx>, params: I)
-> &'tcx Substs<'tcx>
where I: IntoIterator<Item=Kind<'tcx>> {
tcx.mk_substs(params.into_iter().collect())
tcx.mk_substs(&params.into_iter().collect::<Vec<_>>())
}

pub fn maybe_new<I, E>(tcx: TyCtxt<'a, 'gcx, 'tcx>, params: I)
Expand Down Expand Up @@ -304,8 +304,8 @@ impl<'a, 'gcx, 'tcx> Substs<'tcx> {

impl<'tcx> TypeFoldable<'tcx> for &'tcx Substs<'tcx> {
fn super_fold_with<'gcx: 'tcx, F: TypeFolder<'gcx, 'tcx>>(&self, folder: &mut F) -> Self {
let params = self.iter().map(|k| k.fold_with(folder)).collect();
folder.tcx().mk_substs(params)
let params = self.iter().map(|k| k.fold_with(folder)).collect::<Vec<_>>();
folder.tcx().mk_substs(&params)
}

fn fold_with<'gcx: 'tcx, F: TypeFolder<'gcx, 'tcx>>(&self, folder: &mut F) -> Self {
Expand Down
10 changes: 5 additions & 5 deletions src/librustc_driver/test.rs
Original file line number Diff line number Diff line change
Expand Up @@ -275,7 +275,7 @@ impl<'a, 'gcx, 'tcx> Env<'a, 'gcx, 'tcx> {
}

pub fn t_pair(&self, ty1: Ty<'tcx>, ty2: Ty<'tcx>) -> Ty<'tcx> {
self.infcx.tcx.mk_tup(vec![ty1, ty2])
self.infcx.tcx.mk_tup(&[ty1, ty2])
}

pub fn t_param(&self, index: u32) -> Ty<'tcx> {
Expand Down Expand Up @@ -803,8 +803,8 @@ fn walk_ty() {
let tcx = env.infcx.tcx;
let int_ty = tcx.types.isize;
let uint_ty = tcx.types.usize;
let tup1_ty = tcx.mk_tup(vec![int_ty, uint_ty, int_ty, uint_ty]);
let tup2_ty = tcx.mk_tup(vec![tup1_ty, tup1_ty, uint_ty]);
let tup1_ty = tcx.mk_tup(&[int_ty, uint_ty, int_ty, uint_ty]);
let tup2_ty = tcx.mk_tup(&[tup1_ty, tup1_ty, uint_ty]);
let uniq_ty = tcx.mk_box(tup2_ty);
let walked: Vec<_> = uniq_ty.walk().collect();
assert_eq!(walked,
Expand All @@ -819,8 +819,8 @@ fn walk_ty_skip_subtree() {
let tcx = env.infcx.tcx;
let int_ty = tcx.types.isize;
let uint_ty = tcx.types.usize;
let tup1_ty = tcx.mk_tup(vec![int_ty, uint_ty, int_ty, uint_ty]);
let tup2_ty = tcx.mk_tup(vec![tup1_ty, tup1_ty, uint_ty]);
let tup1_ty = tcx.mk_tup(&[int_ty, uint_ty, int_ty, uint_ty]);
let tup2_ty = tcx.mk_tup(&[tup1_ty, tup1_ty, uint_ty]);
let uniq_ty = tcx.mk_box(tup2_ty);

// types we expect to see (in order), plus a boolean saying
Expand Down
4 changes: 2 additions & 2 deletions src/librustc_metadata/decoder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -374,7 +374,7 @@ impl<'a, 'tcx> SpecializedDecoder<ty::GenericPredicates<'tcx>> for DecodeContext

impl<'a, 'tcx> SpecializedDecoder<&'tcx Substs<'tcx>> for DecodeContext<'a, 'tcx> {
fn specialized_decode(&mut self) -> Result<&'tcx Substs<'tcx>, Self::Error> {
Ok(self.tcx().mk_substs(Decodable::decode(self)?))
Ok(self.tcx().mk_substs(&Vec::decode(self)?))
}
}

Expand All @@ -386,7 +386,7 @@ impl<'a, 'tcx> SpecializedDecoder<&'tcx ty::Region> for DecodeContext<'a, 'tcx>

impl<'a, 'tcx> SpecializedDecoder<&'tcx ty::Slice<Ty<'tcx>>> for DecodeContext<'a, 'tcx> {
fn specialized_decode(&mut self) -> Result<&'tcx ty::Slice<Ty<'tcx>>, Self::Error> {
Ok(self.tcx().mk_type_list(Decodable::decode(self)?))
Ok(self.tcx().mk_type_list(&Vec::decode(self)?))
}
}

Expand Down
2 changes: 1 addition & 1 deletion src/librustc_mir/build/expr/as_rvalue.rs
Original file line number Diff line number Diff line change
Expand Up @@ -257,7 +257,7 @@ impl<'a, 'gcx, 'tcx> Builder<'a, 'gcx, 'tcx> {
let source_info = self.source_info(span);
let bool_ty = self.hir.bool_ty();
if self.hir.check_overflow() && op.is_checkable() && ty.is_integral() {
let result_tup = self.hir.tcx().mk_tup(vec![ty, bool_ty]);
let result_tup = self.hir.tcx().mk_tup(&[ty, bool_ty]);
let result_value = self.temp(result_tup);

self.cfg.push_assign(block, source_info,
Expand Down
2 changes: 1 addition & 1 deletion src/librustc_trans/callee.rs
Original file line number Diff line number Diff line change
Expand Up @@ -327,7 +327,7 @@ fn trans_fn_pointer_shim<'a, 'tcx>(
}
};
let sig = tcx.erase_late_bound_regions_and_normalize(sig);
let tuple_input_ty = tcx.mk_tup(sig.inputs.to_vec());
let tuple_input_ty = tcx.mk_tup(&sig.inputs);
let sig = ty::FnSig {
inputs: vec![bare_fn_ty_maybe_ref,
tuple_input_ty],
Expand Down
2 changes: 1 addition & 1 deletion src/librustc_trans/mir/constant.rs
Original file line number Diff line number Diff line change
Expand Up @@ -773,7 +773,7 @@ impl<'a, 'tcx> MirConstContext<'a, 'tcx> {
let rhs = self.const_operand(rhs, span)?;
let ty = lhs.ty;
let val_ty = op.ty(tcx, lhs.ty, rhs.ty);
let binop_ty = tcx.mk_tup(vec![val_ty, tcx.types.bool]);
let binop_ty = tcx.mk_tup(&[val_ty, tcx.types.bool]);
let (lhs, rhs) = (lhs.llval, rhs.llval);
assert!(!ty.is_fp());

Expand Down
2 changes: 1 addition & 1 deletion src/librustc_trans/mir/rvalue.rs
Original file line number Diff line number Diff line change
Expand Up @@ -430,7 +430,7 @@ impl<'bcx, 'tcx> MirContext<'bcx, 'tcx> {
lhs.immediate(), rhs.immediate(),
lhs.ty);
let val_ty = op.ty(bcx.tcx(), lhs.ty, rhs.ty);
let operand_ty = bcx.tcx().mk_tup(vec![val_ty, bcx.tcx().types.bool]);
let operand_ty = bcx.tcx().mk_tup(&[val_ty, bcx.tcx().types.bool]);
let operand = OperandRef {
val: result,
ty: operand_ty
Expand Down
6 changes: 3 additions & 3 deletions src/librustc_typeck/astconv.rs
Original file line number Diff line number Diff line change
Expand Up @@ -660,7 +660,7 @@ impl<'o, 'gcx: 'tcx, 'tcx> AstConv<'gcx, 'tcx>+'o {
span: output_span
};

(self.tcx().mk_tup(inputs), output_binding)
(self.tcx().mk_tup(&inputs), output_binding)
}

pub fn instantiate_poly_trait_ref(&self,
Expand Down Expand Up @@ -1663,8 +1663,8 @@ impl<'o, 'gcx: 'tcx, 'tcx> AstConv<'gcx, 'tcx>+'o {
hir::TyTup(ref fields) => {
let flds = fields.iter()
.map(|t| self.ast_ty_to_ty(rscope, &t))
.collect();
tcx.mk_tup(flds)
.collect::<Vec<_>>();
tcx.mk_tup(&flds)
}
hir::TyBareFn(ref bf) => {
require_c_abi_if_variadic(tcx, &bf.decl, bf.abi, ast_ty.span);
Expand Down
2 changes: 1 addition & 1 deletion src/librustc_typeck/check/_match.rs
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,7 @@ impl<'a, 'gcx, 'tcx> FnCtxt<'a, 'gcx, 'tcx> {
let max_len = cmp::max(expected_len, elements.len());

let element_tys: Vec<_> = (0 .. max_len).map(|_| self.next_ty_var()).collect();
let pat_ty = tcx.mk_tup(element_tys.clone());
let pat_ty = tcx.mk_tup(&element_tys);
self.demand_eqtype(pat.span, expected, pat_ty);
for (i, elem) in elements.iter().enumerate_and_adjust(max_len, ddpos) {
self.check_pat(elem, &element_tys[i]);
Expand Down
4 changes: 2 additions & 2 deletions src/librustc_typeck/check/closure.rs
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ impl<'a, 'gcx, 'tcx> FnCtxt<'a, 'gcx, 'tcx> {

let closure_type = self.tcx.mk_closure(expr_def_id,
self.parameter_environment.free_substs,
upvar_tys);
&upvar_tys);

let fn_sig = self.tcx
.liberate_late_bound_regions(self.tcx.region_maps.call_site_extent(expr.id, body.id),
Expand All @@ -88,7 +88,7 @@ impl<'a, 'gcx, 'tcx> FnCtxt<'a, 'gcx, 'tcx> {

// Tuple up the arguments and insert the resulting function type into
// the `closures` table.
fn_ty.sig.0.inputs = vec![self.tcx.mk_tup(fn_ty.sig.0.inputs)];
fn_ty.sig.0.inputs = vec![self.tcx.mk_tup(&fn_ty.sig.0.inputs)];

debug!("closure for {:?} --> sig={:?} opt_kind={:?}",
expr_def_id,
Expand Down
Loading

0 comments on commit ce9e95a

Please sign in to comment.