Skip to content

Commit 2e80d68

Browse files
ijlijl
committed
unlikely to cold_path, remove intrinsics
1 parent 27edea9 commit 2e80d68

File tree

24 files changed

+108
-115
lines changed

24 files changed

+108
-115
lines changed

‎Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,9 +32,9 @@ unwind = ["unwinding"]
3232

3333
# Features detected by build.rs. Do not specify.
3434
avx512 = []
35+
cold_path = []
3536
generic_simd = []
3637
inline_int = []
37-
intrinsics = []
3838
optimize = []
3939

4040
[dependencies]

‎build.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,9 @@ fn main() {
77
println!("cargo:rerun-if-env-changed=CFLAGS");
88
println!("cargo:rerun-if-env-changed=LDFLAGS");
99
println!("cargo:rerun-if-env-changed=RUSTFLAGS");
10+
println!("cargo:rustc-check-cfg=cfg(cold_path)");
1011
println!("cargo:rustc-check-cfg=cfg(CPython)");
1112
println!("cargo:rustc-check-cfg=cfg(GraalPy)");
12-
println!("cargo:rustc-check-cfg=cfg(intrinsics)");
1313
println!("cargo:rustc-check-cfg=cfg(optimize)");
1414
println!("cargo:rustc-check-cfg=cfg(Py_3_10)");
1515
println!("cargo:rustc-check-cfg=cfg(Py_3_11)");
@@ -50,8 +50,8 @@ fn main() {
5050
println!("cargo:rustc-cfg=feature=\"avx512\"");
5151
}
5252

53-
if version_check::supports_feature("core_intrinsics").unwrap_or(false) {
54-
println!("cargo:rustc-cfg=feature=\"intrinsics\"");
53+
if version_check::supports_feature("cold_path").unwrap_or(false) {
54+
println!("cargo:rustc-cfg=feature=\"cold_path\"");
5555
}
5656

5757
if version_check::supports_feature("optimize_attribute").unwrap_or(false) {

‎src/deserialize/backend/yyjson.rs‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ pub(crate) fn deserialize(
7373
assume!(!data.is_empty());
7474
let buffer_capacity = buffer_capacity_to_allocate(data.len());
7575
let buffer_ptr = ffi!(PyMem_Malloc(buffer_capacity));
76-
if unlikely!(buffer_ptr.is_null()) {
76+
if buffer_ptr.is_null() {
7777
return Err(DeserializeError::from_yyjson(
7878
Cow::Borrowed("Not enough memory to allocate buffer for parsing"),
7979
0,
@@ -104,14 +104,15 @@ pub(crate) fn deserialize(
104104
&raw mut err,
105105
)
106106
};
107-
if unlikely!(doc.is_null()) {
107+
if doc.is_null() {
108108
ffi!(PyMem_Free(buffer_ptr));
109109
let msg: Cow<str> = unsafe { core::ffi::CStr::from_ptr(err.msg).to_string_lossy() };
110110
return Err(DeserializeError::from_yyjson(msg, err.pos as i64, data));
111111
}
112112
let val = yyjson_doc_get_root(doc);
113113
let pyval = {
114-
if unlikely!(!unsafe_yyjson_is_ctn(val)) {
114+
if !unsafe_yyjson_is_ctn(val) {
115+
cold_path!();
115116
match ElementType::from_tag(val) {
116117
ElementType::String => parse_yy_string(val),
117118
ElementType::Uint64 => parse_yy_u64(val),
@@ -214,7 +215,8 @@ fn populate_yy_array(list: *mut crate::ffi::PyObject, elem: *mut yyjson_val) {
214215

215216
for _ in 0..len {
216217
let val = next;
217-
if unlikely!(unsafe_yyjson_is_ctn(val)) {
218+
if unsafe_yyjson_is_ctn(val) {
219+
cold_path!();
218220
next = unsafe_yyjson_get_next_container(val);
219221
if is_yyjson_tag!(val, TAG_ARRAY) {
220222
let pyval = ffi!(PyList_New(usize_to_isize(unsafe_yyjson_get_len(val))));
@@ -265,7 +267,8 @@ fn populate_yy_object(dict: *mut crate::ffi::PyObject, elem: *mut yyjson_val) {
265267
);
266268
get_unicode_key(key_str)
267269
};
268-
if unlikely!(unsafe_yyjson_is_ctn(val)) {
270+
if unsafe_yyjson_is_ctn(val) {
271+
cold_path!();
269272
next_key = unsafe_yyjson_get_next_container(val);
270273
next_val = next_key.add(1);
271274
if is_yyjson_tag!(val, TAG_ARRAY) {

‎src/deserialize/deserializer.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,8 @@ pub(crate) fn deserialize(
1212
let buffer = read_input_to_buf(ptr)?;
1313
debug_assert!(!buffer.is_empty());
1414

15-
if unlikely!(buffer.len() == 2) {
15+
if buffer.len() == 2 {
16+
cold_path!();
1617
if buffer == b"[]" {
1718
return Ok(nonnull!(ffi!(PyList_New(0))));
1819
} else if buffer == b"{}" {

‎src/deserialize/pyobject.rs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,8 @@ use core::ptr::NonNull;
99
#[cfg(not(Py_GIL_DISABLED))]
1010
#[inline(always)]
1111
pub(crate) fn get_unicode_key(key_str: &str) -> PyStr {
12-
if unlikely!(key_str.len() > 64) {
12+
if key_str.len() > 64 {
13+
cold_path!();
1314
PyStr::from_str_with_hash(key_str)
1415
} else {
1516
assume!(key_str.len() <= 64);
@@ -23,7 +24,7 @@ pub(crate) fn get_unicode_key(key_str: &str) -> PyStr {
2324
|| hash,
2425
|| CachedKey::new(PyStr::from_str_with_hash(key_str)),
2526
);
26-
unsafe { entry.get() }
27+
entry.get()
2728
}
2829
}
2930
}

‎src/deserialize/utf8.rs‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -51,12 +51,13 @@ pub(crate) fn read_input_to_buf(
5151
} else if is_type!(obj_type_ptr, STR_TYPE) {
5252
let pystr = unsafe { PyStr::from_ptr_unchecked(ptr) };
5353
let uni = pystr.to_str();
54-
if unlikely!(uni.is_none()) {
54+
if uni.is_none() {
5555
return Err(DeserializeError::invalid(Cow::Borrowed(INVALID_STR)));
5656
}
5757
let as_str = uni.unwrap();
5858
buffer = unsafe { core::slice::from_raw_parts(as_str.as_ptr(), as_str.len()) };
59-
} else if unlikely!(is_type!(obj_type_ptr, MEMORYVIEW_TYPE)) {
59+
} else if is_type!(obj_type_ptr, MEMORYVIEW_TYPE) {
60+
cold_path!();
6061
let membuf = unsafe { PyMemoryView_GET_BUFFER(ptr) };
6162
if unsafe { crate::ffi::PyBuffer_IsContiguous(membuf, b'C' as c_char) == 0 } {
6263
return Err(DeserializeError::invalid(Cow::Borrowed(
@@ -72,7 +73,8 @@ pub(crate) fn read_input_to_buf(
7273
if !is_valid_utf8(buffer) {
7374
return Err(DeserializeError::invalid(Cow::Borrowed(INVALID_STR)));
7475
}
75-
} else if unlikely!(is_type!(obj_type_ptr, BYTEARRAY_TYPE)) {
76+
} else if is_type!(obj_type_ptr, BYTEARRAY_TYPE) {
77+
cold_path!();
7678
buffer = unsafe {
7779
core::slice::from_raw_parts(
7880
ffi!(PyByteArray_AsString(ptr)).cast::<u8>().cast_const(),
@@ -87,7 +89,8 @@ pub(crate) fn read_input_to_buf(
8789
"Input must be bytes, bytearray, memoryview, or str",
8890
)));
8991
}
90-
if unlikely!(buffer.is_empty()) {
92+
if buffer.is_empty() {
93+
cold_path!();
9194
Err(DeserializeError::invalid(Cow::Borrowed(
9295
"Input is a zero-length, empty document",
9396
)))

‎src/lib.rs‎

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,8 @@
11
// SPDX-License-Identifier: (Apache-2.0 OR MIT)
22

3-
#![cfg_attr(feature = "intrinsics", feature(core_intrinsics))]
43
#![cfg_attr(feature = "optimize", feature(optimize_attribute))]
54
#![cfg_attr(feature = "generic_simd", feature(portable_simd))]
6-
#![allow(internal_features)] // core_intrinsics
5+
#![cfg_attr(feature = "cold_path", feature(cold_path))]
76
#![allow(non_camel_case_types)]
87
#![allow(stable_features)] // MSRV
98
#![allow(static_mut_refs)]
@@ -392,7 +391,8 @@ pub(crate) unsafe extern "C" fn dumps(
392391
let mut optsptr: Option<NonNull<PyObject>> = None;
393392

394393
let num_args = PyVectorcall_NARGS(isize_to_usize(nargs));
395-
if unlikely!(num_args == 0) {
394+
if num_args == 0 {
395+
cold_path!();
396396
return raise_dumps_exception_fixed(
397397
"dumps() missing 1 required positional argument: 'obj'",
398398
);
@@ -403,18 +403,21 @@ pub(crate) unsafe extern "C" fn dumps(
403403
if num_args & 3 == 3 {
404404
optsptr = Some(NonNull::new_unchecked(*args.offset(2)));
405405
}
406-
if unlikely!(!kwnames.is_null()) {
406+
if !kwnames.is_null() {
407+
cold_path!();
407408
for i in 0..=Py_SIZE(kwnames).saturating_sub(1) {
408409
let arg = crate::ffi::PyTuple_GET_ITEM(kwnames, i as Py_ssize_t);
409410
if core::ptr::eq(arg, typeref::DEFAULT) {
410-
if unlikely!(num_args & 2 == 2) {
411+
if num_args & 2 == 2 {
412+
cold_path!();
411413
return raise_dumps_exception_fixed(
412414
"dumps() got multiple values for argument: 'default'",
413415
);
414416
}
415417
default = Some(NonNull::new_unchecked(*args.offset(num_args + i)));
416418
} else if core::ptr::eq(arg, typeref::OPTION) {
417-
if unlikely!(num_args & 3 == 3) {
419+
if num_args & 3 == 3 {
420+
cold_path!();
418421
return raise_dumps_exception_fixed(
419422
"dumps() got multiple values for argument: 'option'",
420423
);
@@ -430,14 +433,17 @@ pub(crate) unsafe extern "C" fn dumps(
430433

431434
let mut optsbits: i32 = 0;
432435
if let Some(opts) = optsptr {
436+
cold_path!();
433437
if core::ptr::eq((*opts.as_ptr()).ob_type, typeref::INT_TYPE) {
434438
#[allow(clippy::cast_possible_truncation)]
435439
let tmp = PyLong_AsLong(optsptr.unwrap().as_ptr()) as i32; // stmt_expr_attributes
436440
optsbits = tmp;
437-
if unlikely!(!(0..=opt::MAX_OPT).contains(&optsbits)) {
441+
if !(0..=opt::MAX_OPT).contains(&optsbits) {
442+
cold_path!();
438443
return raise_dumps_exception_fixed("Invalid opts");
439444
}
440-
} else if unlikely!(!core::ptr::eq(opts.as_ptr(), typeref::NONE)) {
445+
} else if !core::ptr::eq(opts.as_ptr(), typeref::NONE) {
446+
cold_path!();
441447
return raise_dumps_exception_fixed("Invalid opts");
442448
}
443449
}

‎src/serialize/obtype.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,8 @@ pub(crate) fn pyobject_to_obtype_unlikely(
101101
return ObType::Dataclass;
102102
}
103103

104-
if unlikely!(opt_enabled!(opts, SERIALIZE_NUMPY)) {
104+
if opt_enabled!(opts, SERIALIZE_NUMPY) {
105+
cold_path!();
105106
if is_numpy_scalar(ob_type) {
106107
return ObType::NumpyScalar;
107108
} else if is_numpy_array(ob_type) {

‎src/serialize/per_type/dataclass.rs‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -31,12 +31,13 @@ impl Serialize for DataclassGenericSerializer<'_> {
3131
where
3232
S: Serializer,
3333
{
34-
if unlikely!(self.previous.state.recursion_limit()) {
34+
if self.previous.state.recursion_limit() {
3535
err!(SerializeError::RecursionLimit)
3636
}
3737
let dict = ffi!(PyObject_GetAttr(self.previous.ptr, DICT_STR));
3838
let ob_type = ob_type!(self.previous.ptr);
39-
if unlikely!(dict.is_null()) {
39+
if dict.is_null() {
40+
cold_path!();
4041
ffi!(PyErr_Clear());
4142
DataclassFallbackSerializer::new(
4243
self.previous.ptr,
@@ -90,7 +91,8 @@ impl Serialize for DataclassFastSerializer {
9091
S: Serializer,
9192
{
9293
let len = isize_to_usize(ffi!(Py_SIZE(self.ptr)));
93-
if unlikely!(len == 0) {
94+
if len == 0 {
95+
cold_path!();
9496
return ZeroDictSerializer::new().serialize(serializer);
9597
}
9698
let mut map = serializer.serialize_map(None).unwrap();
@@ -109,15 +111,17 @@ impl Serialize for DataclassFastSerializer {
109111

110112
let key_as_str = {
111113
let key_ob_type = ob_type!(key);
112-
if unlikely!(!is_class_by_type!(key_ob_type, STR_TYPE)) {
114+
if !is_class_by_type!(key_ob_type, STR_TYPE) {
115+
cold_path!();
113116
err!(SerializeError::KeyMustBeStr)
114117
}
115118
match unsafe { PyStr::from_ptr_unchecked(key).to_str() } {
116119
Some(uni) => uni,
117120
None => err!(SerializeError::InvalidStr),
118121
}
119122
};
120-
if unlikely!(key_as_str.as_bytes()[0] == b'_') {
123+
if key_as_str.as_bytes()[0] == b'_' {
124+
cold_path!();
121125
continue;
122126
}
123127
let pyvalue = PyObjectSerializer::new(value, self.state, self.default);
@@ -159,7 +163,8 @@ impl Serialize for DataclassFallbackSerializer {
159163
debug_assert!(ffi!(Py_REFCNT(fields)) >= 2);
160164
ffi!(Py_DECREF(fields));
161165
let len = isize_to_usize(ffi!(Py_SIZE(fields)));
162-
if unlikely!(len == 0) {
166+
if len == 0 {
167+
cold_path!();
163168
return ZeroDictSerializer::new().serialize(serializer);
164169
}
165170
let mut map = serializer.serialize_map(None).unwrap();
@@ -189,6 +194,7 @@ impl Serialize for DataclassFallbackSerializer {
189194
None => err!(SerializeError::InvalidStr),
190195
};
191196
if key_as_str.as_bytes()[0] == b'_' {
197+
cold_path!();
192198
continue;
193199
}
194200

‎src/serialize/per_type/datetime.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,8 @@ impl Date {
4848
let year = ffi!(PyDateTime_GET_YEAR(self.ptr));
4949
let mut yearbuf = itoa::Buffer::new();
5050
let formatted = yearbuf.format(year);
51-
if unlikely!(year < 1000) {
51+
if year < 1000 {
52+
cold_path!();
5253
// date-fullyear = 4DIGIT
5354
buf.put_slice(&[b'0', b'0', b'0', b'0'][..(4 - formatted.len())]);
5455
}

0 commit comments

Comments
 (0)