Skip to content

Make ASCII case conversions more than 4× faster #59283

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 11 commits into from
Mar 28, 2019
Merged

Make ASCII case conversions more than 4× faster #59283

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
8740d5d
Add benchmarks for [u8]::make_ascii_uppercase
SimonSapin Mar 18, 2019
ce933f7
Make u8::to_ascii_lowercase and to_ascii_uppercase branchless
SimonSapin Mar 18, 2019
fbe34cc
Add benchmark for not-quite-correct “fake SIMD” make_ascii_uppercase
SimonSapin Mar 18, 2019
e3fb6f8
Tidy
SimonSapin Mar 18, 2019
525a043
Rename src/libcore/benches/ascii_case.rs to ascii.rs
SimonSapin Mar 18, 2019
6d3840b
Add benchmarks for `u8::is_ascii*`
SimonSapin Mar 18, 2019
b4faa9b
Remove ASCII_CHARACTER_CLASS table, use `match` with range patterns i…
SimonSapin Mar 18, 2019
4a3241a
Benchmark more possibles impls of [u8]::make_ascii_uppercase
SimonSapin Mar 18, 2019
0ad91f7
Simplify u8::to_ascii_{upp,low}ercase while keeping it fast
SimonSapin Mar 18, 2019
c1ec29a
ASCII uppercase: add "subtract shifted bool" benchmark
SimonSapin Mar 19, 2019
7fad370
ASCII uppercase: add "subtract multiplied bool" benchmark
SimonSapin Mar 19, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Make u8::to_ascii_lowercase and to_ascii_uppercase branchless
  • Loading branch information
@SimonSapin
SimonSapin committed Mar 18, 2019
commit ce933f77c865a15670855ac5941fe200752b739f
22 changes: 1 addition & 21 deletions src/libcore/benches/ascii_case.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,24 +1,4 @@
// Lower-case ASCII 'a' is the first byte that has its highest bit set after wrap-adding 0x1F:
//
// b'a' + 0x1F == 0x80 == 0b1000_0000
// b'z' + 0x1F == 0x98 == 0b10011000
//
// Lower-case ASCII 'z' is the last byte that has its highest bit unset after wrap-adding 0x05:
//
// b'a' + 0x05 == 0x66 == 0b0110_0110
// b'z' + 0x05 == 0x7F == 0b0111_1111
//
// … except for 0xFB to 0xFF, but those are in the range of bytes that have the highest bit
// unset again after adding 0x1F.
//
// So `(byte + 0x1f) & !(byte + 5)` has its highest bit set
// iff `byte` is a lower-case ASCII letter.
//
// Lower-case ASCII letters all have the 0x20 bit set.
// (Two positions right of 0x80, the highest bit.)
// Unsetting that bit produces the same letter, in upper-case.
//
// Therefore:
// See comments in `u8::to_ascii_uppercase` in `src/libcore/num/mod.rs`.
fn branchless_to_ascii_upper_case(byte: u8) -> u8 {
byte &
!(
Expand Down
122 changes: 42 additions & 80 deletions src/libcore/num/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3794,7 +3794,39 @@ impl u8 {
#[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")]
#[inline]
pub fn to_ascii_uppercase(&self) -> u8 {
ASCII_UPPERCASE_MAP[*self as usize]
// See benchmarks in src/libcore/benches/ascii_case.rs

// Lower-case ASCII 'a' is the first byte that has its highest bit set
// after wrap-adding 0x1F:
//
// b'a' + 0x1F == 0x80 == 0b1000_0000
// b'z' + 0x1F == 0x98 == 0b10011000
//
// Lower-case ASCII 'z' is the last byte that has its highest bit unset
// after wrap-adding 0x05:
//
// b'a' + 0x05 == 0x66 == 0b0110_0110
// b'z' + 0x05 == 0x7F == 0b0111_1111
//
// … except for 0xFB to 0xFF, but those are in the range of bytes
// that have the highest bit unset again after adding 0x1F.
//
// So `(byte + 0x1f) & !(byte + 5)` has its highest bit set
// iff `byte` is a lower-case ASCII letter.
//
// Lower-case ASCII letters all have the 0x20 bit set.
// (Two positions right of 0x80, the highest bit.)
// Unsetting that bit produces the same letter, in upper-case.
//
// Therefore:
*self &
!(
(
self.wrapping_add(0x1f) &
!self.wrapping_add(0x05) &
0x80
) >> 2
)
}

/// Makes a copy of the value in its ASCII lower case equivalent.
Expand All @@ -3816,7 +3848,15 @@ impl u8 {
#[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")]
#[inline]
pub fn to_ascii_lowercase(&self) -> u8 {
ASCII_LOWERCASE_MAP[*self as usize]
// See comments in to_ascii_uppercase above.
*self |
(
(
self.wrapping_add(0x3f) &
!self.wrapping_add(0x25) &
0x80
) >> 2
)
}

/// Checks that two values are an ASCII case-insensitive match.
Expand Down Expand Up @@ -4940,84 +4980,6 @@ impl_from! { u32, f64, #[stable(feature = "lossless_float_conv", since = "1.6.0"
// Float -> Float
impl_from! { f32, f64, #[stable(feature = "lossless_float_conv", since = "1.6.0")] }

static ASCII_LOWERCASE_MAP: [u8; 256] = [
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
b' ', b'!', b'"', b'#', b'$', b'%', b'&', b'\'',
b'(', b')', b'*', b'+', b',', b'-', b'.', b'/',
b'0', b'1', b'2', b'3', b'4', b'5', b'6', b'7',
b'8', b'9', b':', b';', b'<', b'=', b'>', b'?',
b'@',

b'a', b'b', b'c', b'd', b'e', b'f', b'g',
b'h', b'i', b'j', b'k', b'l', b'm', b'n', b'o',
b'p', b'q', b'r', b's', b't', b'u', b'v', b'w',
b'x', b'y', b'z',

b'[', b'\\', b']', b'^', b'_',
b'`', b'a', b'b', b'c', b'd', b'e', b'f', b'g',
b'h', b'i', b'j', b'k', b'l', b'm', b'n', b'o',
b'p', b'q', b'r', b's', b't', b'u', b'v', b'w',
b'x', b'y', b'z', b'{', b'|', b'}', b'~', 0x7f,
0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f,
0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7,
0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf,
0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7,
0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf,
0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7,
0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf,
0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7,
0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf,
0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7,
0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef,
0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7,
0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff,
];

static ASCII_UPPERCASE_MAP: [u8; 256] = [
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
b' ', b'!', b'"', b'#', b'$', b'%', b'&', b'\'',
b'(', b')', b'*', b'+', b',', b'-', b'.', b'/',
b'0', b'1', b'2', b'3', b'4', b'5', b'6', b'7',
b'8', b'9', b':', b';', b'<', b'=', b'>', b'?',
b'@', b'A', b'B', b'C', b'D', b'E', b'F', b'G',
b'H', b'I', b'J', b'K', b'L', b'M', b'N', b'O',
b'P', b'Q', b'R', b'S', b'T', b'U', b'V', b'W',
b'X', b'Y', b'Z', b'[', b'\\', b']', b'^', b'_',
b'`',

b'A', b'B', b'C', b'D', b'E', b'F', b'G',
b'H', b'I', b'J', b'K', b'L', b'M', b'N', b'O',
b'P', b'Q', b'R', b'S', b'T', b'U', b'V', b'W',
b'X', b'Y', b'Z',

b'{', b'|', b'}', b'~', 0x7f,
0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f,
0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7,
0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf,
0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7,
0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf,
0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7,
0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf,
0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7,
0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf,
0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7,
0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef,
0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7,
0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff,
];

enum AsciiCharacterClass {
C, // control
Cw, // control whitespace
Expand Down