Skip to content

Commit

Permalink
Unrolled build for rust-lang#122884
Browse files Browse the repository at this point in the history 
Rollup merge of rust-lang#122884 - mzabaluev:pow-remove-exit-branch, r=Amanieu

Optimize integer `pow` by removing the exit branch

The branch at the end of the `pow` implementations is redundant with multiplication code already present in the loop. By rotating the exit check, this branch can be largely removed, improving code size and reducing instruction cache misses.

Testing on my machine (`x86_64`, 11th Gen Intel Core i5-1135G7 @ 2.40GHz), the `num::int_pow` benchmarks improve by some 40% for the unchecked operations and show some slight improvement for the checked operations as well.
  • Loading branch information
@rust-timer
rust-timer authored Aug 14, 2024
2 parents 9859bf2 + ac88b33 commit f377c6a
Show file tree
Hide file tree
Showing 3 changed files with 151 additions and 89 deletions.
1 change: 1 addition & 0 deletions library/core/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,7 @@
#![feature(internal_impls_macro)]
#![feature(ip)]
#![feature(is_ascii_octdigit)]
#![feature(is_val_statically_known)]
#![feature(isqrt)]
#![feature(link_cfg)]
#![feature(offset_of_enum)]
Expand Down
118 changes: 75 additions & 43 deletions library/core/src/num/int_macros.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1496,18 +1496,17 @@ macro_rules! int_impl {
let mut base = self;
let mut acc: Self = 1;

while exp > 1 {
loop {
if (exp & 1) == 1 {
acc = try_opt!(acc.checked_mul(base));
// since exp!=0, finally the exp must be 1.
if exp == 1 {
return Some(acc);
}
}
exp /= 2;
base = try_opt!(base.checked_mul(base));
}
// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary and may cause a
// needless overflow.
acc.checked_mul(base)
}

/// Strict exponentiation. Computes `self.pow(exp)`, panicking if
Expand Down Expand Up @@ -1547,18 +1546,17 @@ macro_rules! int_impl {
let mut base = self;
let mut acc: Self = 1;

while exp > 1 {
loop {
if (exp & 1) == 1 {
acc = acc.strict_mul(base);
// since exp!=0, finally the exp must be 1.
if exp == 1 {
return acc;
}
}
exp /= 2;
base = base.strict_mul(base);
}
// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary and may cause a
// needless overflow.
acc.strict_mul(base)
}

/// Returns the square root of the number, rounded down.
Expand Down Expand Up @@ -2175,26 +2173,44 @@ macro_rules! int_impl {
#[must_use = "this returns the result of the operation, \
without modifying the original"]
#[inline]
#[rustc_allow_const_fn_unstable(is_val_statically_known)]
pub const fn wrapping_pow(self, mut exp: u32) -> Self {
if exp == 0 {
return 1;
}
let mut base = self;
let mut acc: Self = 1;

while exp > 1 {
if (exp & 1) == 1 {
acc = acc.wrapping_mul(base);
if intrinsics::is_val_statically_known(exp) {
while exp > 1 {
if (exp & 1) == 1 {
acc = acc.wrapping_mul(base);
}
exp /= 2;
base = base.wrapping_mul(base);
}
exp /= 2;
base = base.wrapping_mul(base);
}

// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary and may cause a
// needless overflow.
acc.wrapping_mul(base)
// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary.
acc.wrapping_mul(base)
} else {
// This is faster than the above when the exponent is not known
// at compile time. We can't use the same code for the constant
// exponent case because LLVM is currently unable to unroll
// this loop.
loop {
if (exp & 1) == 1 {
acc = acc.wrapping_mul(base);
// since exp!=0, finally the exp must be 1.
if exp == 1 {
return acc;
}
}
exp /= 2;
base = base.wrapping_mul(base);
}
}
}

/// Calculates `self` + `rhs`.
Expand Down Expand Up @@ -2690,9 +2706,14 @@ macro_rules! int_impl {
// Scratch space for storing results of overflowing_mul.
let mut r;

while exp > 1 {
loop {
if (exp & 1) == 1 {
r = acc.overflowing_mul(base);
// since exp!=0, finally the exp must be 1.
if exp == 1 {
r.1 |= overflown;
return r;
}
acc = r.0;
overflown |= r.1;
}
Expand All @@ -2701,14 +2722,6 @@ macro_rules! int_impl {
base = r.0;
overflown |= r.1;
}

// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary and may cause a
// needless overflow.
r = acc.overflowing_mul(base);
r.1 |= overflown;
r
}

/// Raises self to the power of `exp`, using exponentiation by squaring.
Expand All @@ -2728,26 +2741,45 @@ macro_rules! int_impl {
without modifying the original"]
#[inline]
#[rustc_inherit_overflow_checks]
#[rustc_allow_const_fn_unstable(is_val_statically_known)]
pub const fn pow(self, mut exp: u32) -> Self {
if exp == 0 {
return 1;
}
let mut base = self;
let mut acc = 1;

while exp > 1 {
if (exp & 1) == 1 {
acc = acc * base;
if intrinsics::is_val_statically_known(exp) {
while exp > 1 {
if (exp & 1) == 1 {
acc = acc * base;
}
exp /= 2;
base = base * base;
}
exp /= 2;
base = base * base;
}

// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary and may cause a
// needless overflow.
acc * base
// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary and may cause a
// needless overflow.
acc * base
} else {
// This is faster than the above when the exponent is not known
// at compile time. We can't use the same code for the constant
// exponent case because LLVM is currently unable to unroll
// this loop.
loop {
if (exp & 1) == 1 {
acc = acc * base;
// since exp!=0, finally the exp must be 1.
if exp == 1 {
return acc;
}
}
exp /= 2;
base = base * base;
}
}
}

/// Returns the square root of the number, rounded down.
Expand Down
121 changes: 75 additions & 46 deletions library/core/src/num/uint_macros.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1622,20 +1622,17 @@ macro_rules! uint_impl {
let mut base = self;
let mut acc: Self = 1;

while exp > 1 {
loop {
if (exp & 1) == 1 {
acc = try_opt!(acc.checked_mul(base));
// since exp!=0, finally the exp must be 1.
if exp == 1 {
return Some(acc);
}
}
exp /= 2;
base = try_opt!(base.checked_mul(base));
}

// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary and may cause a
// needless overflow.

acc.checked_mul(base)
}

/// Strict exponentiation. Computes `self.pow(exp)`, panicking if
Expand Down Expand Up @@ -1675,18 +1672,17 @@ macro_rules! uint_impl {
let mut base = self;
let mut acc: Self = 1;

while exp > 1 {
loop {
if (exp & 1) == 1 {
acc = acc.strict_mul(base);
// since exp!=0, finally the exp must be 1.
if exp == 1 {
return acc;
}
}
exp /= 2;
base = base.strict_mul(base);
}
// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary and may cause a
// needless overflow.
acc.strict_mul(base)
}

/// Saturating integer addition. Computes `self + rhs`, saturating at
Expand Down Expand Up @@ -2138,26 +2134,44 @@ macro_rules! uint_impl {
#[must_use = "this returns the result of the operation, \
without modifying the original"]
#[inline]
#[rustc_allow_const_fn_unstable(is_val_statically_known)]
pub const fn wrapping_pow(self, mut exp: u32) -> Self {
if exp == 0 {
return 1;
}
let mut base = self;
let mut acc: Self = 1;

while exp > 1 {
if (exp & 1) == 1 {
acc = acc.wrapping_mul(base);
if intrinsics::is_val_statically_known(exp) {
while exp > 1 {
if (exp & 1) == 1 {
acc = acc.wrapping_mul(base);
}
exp /= 2;
base = base.wrapping_mul(base);
}
exp /= 2;
base = base.wrapping_mul(base);
}

// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary and may cause a
// needless overflow.
acc.wrapping_mul(base)
// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary.
acc.wrapping_mul(base)
} else {
// This is faster than the above when the exponent is not known
// at compile time. We can't use the same code for the constant
// exponent case because LLVM is currently unable to unroll
// this loop.
loop {
if (exp & 1) == 1 {
acc = acc.wrapping_mul(base);
// since exp!=0, finally the exp must be 1.
if exp == 1 {
return acc;
}
}
exp /= 2;
base = base.wrapping_mul(base);
}
}
}

/// Calculates `self` + `rhs`.
Expand Down Expand Up @@ -2603,9 +2617,14 @@ macro_rules! uint_impl {
// Scratch space for storing results of overflowing_mul.
let mut r;

while exp > 1 {
loop {
if (exp & 1) == 1 {
r = acc.overflowing_mul(base);
// since exp!=0, finally the exp must be 1.
if exp == 1 {
r.1 |= overflown;
return r;
}
acc = r.0;
overflown |= r.1;
}
Expand All @@ -2614,15 +2633,6 @@ macro_rules! uint_impl {
base = r.0;
overflown |= r.1;
}

// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary and may cause a
// needless overflow.
r = acc.overflowing_mul(base);
r.1 |= overflown;

r
}

/// Raises self to the power of `exp`, using exponentiation by squaring.
Expand All @@ -2640,26 +2650,45 @@ macro_rules! uint_impl {
without modifying the original"]
#[inline]
#[rustc_inherit_overflow_checks]
#[rustc_allow_const_fn_unstable(is_val_statically_known)]
pub const fn pow(self, mut exp: u32) -> Self {
if exp == 0 {
return 1;
}
let mut base = self;
let mut acc = 1;

while exp > 1 {
if (exp & 1) == 1 {
acc = acc * base;
if intrinsics::is_val_statically_known(exp) {
while exp > 1 {
if (exp & 1) == 1 {
acc = acc * base;
}
exp /= 2;
base = base * base;
}
exp /= 2;
base = base * base;
}

// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary and may cause a
// needless overflow.
acc * base
// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary and may cause a
// needless overflow.
acc * base
} else {
// This is faster than the above when the exponent is not known
// at compile time. We can't use the same code for the constant
// exponent case because LLVM is currently unable to unroll
// this loop.
loop {
if (exp & 1) == 1 {
acc = acc * base;
// since exp!=0, finally the exp must be 1.
if exp == 1 {
return acc;
}
}
exp /= 2;
base = base * base;
}
}
}

/// Returns the square root of the number, rounded down.
Expand Down

0 comments on commit f377c6a

Please sign in to comment.