Skip to content

Commit 5b93bdd

Browse files
vstinnervstinner
committed
gh-110481: Move non-limited Py_INCREF() to cpython/object.h
Move the non-limited implementation of Py_INCREF() and Py_DECREF() from Include/object.h to Include/cpython/object.h. Move static inline functions using Py_INCREF/DECREF() after the cpython/object.h include: Py_XINCREF(), Py_XDECREF(), Py_NewRef() and Py_XNewRef().
1 parent 8eaa206 commit 5b93bdd

File tree

2 files changed

+212
-131
lines changed

2 files changed

+212
-131
lines changed

‎Include/cpython/object.h

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,158 @@
22
# error "this header file must not be included directly"
33
#endif
44

5+
#if defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG))
6+
// Implemented in ../object.h
7+
#else
8+
static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op)
9+
{
10+
#if defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG))
11+
// Stable ABI implements Py_INCREF() as a function call on limited C API
12+
// version 3.12 and newer, and on Python built in debug mode. _Py_IncRef()
13+
// was added to Python 3.10.0a7, use Py_IncRef() on older Python versions.
14+
// Py_IncRef() accepts NULL whereas _Py_IncRef() doesn't.
15+
# if Py_LIMITED_API+0 >= 0x030a00A7
16+
_Py_IncRef(op);
17+
# else
18+
Py_IncRef(op);
19+
# endif
20+
#else
21+
// Non-limited C API and limited C API for Python 3.9 and older access
22+
// directly PyObject.ob_refcnt.
23+
#if defined(Py_NOGIL)
24+
uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local);
25+
uint32_t new_local = local + 1;
26+
if (new_local == 0) {
27+
return;
28+
}
29+
if (_Py_IsOwnedByCurrentThread(op)) {
30+
_Py_atomic_store_uint32_relaxed(&op->ob_ref_local, new_local);
31+
}
32+
else {
33+
_Py_atomic_add_ssize(&op->ob_ref_shared, (1 << _Py_REF_SHARED_SHIFT));
34+
}
35+
#elif SIZEOF_VOID_P > 4
36+
// Portable saturated add, branching on the carry flag and set low bits
37+
PY_UINT32_T cur_refcnt = op->ob_refcnt_split[PY_BIG_ENDIAN];
38+
PY_UINT32_T new_refcnt = cur_refcnt + 1;
39+
if (new_refcnt == 0) {
40+
return;
41+
}
42+
op->ob_refcnt_split[PY_BIG_ENDIAN] = new_refcnt;
43+
#else
44+
// Explicitly check immortality against the immortal value
45+
if (_Py_IsImmortal(op)) {
46+
return;
47+
}
48+
op->ob_refcnt++;
49+
#endif
50+
_Py_INCREF_STAT_INC();
51+
#ifdef Py_REF_DEBUG
52+
_Py_INCREF_IncRefTotal();
53+
#endif
54+
#endif
55+
}
56+
57+
# if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000
58+
# define Py_INCREF(op) Py_INCREF(_PyObject_CAST(op))
59+
# endif
60+
#endif
61+
62+
63+
#ifdef Py_NOGIL
64+
// Implements Py_DECREF on objects not owned by the current thread.
65+
PyAPI_FUNC(void) _Py_DecRefShared(PyObject *);
66+
PyAPI_FUNC(void) _Py_DecRefSharedDebug(PyObject *, const char *, int);
67+
68+
// Called from Py_DECREF by the owning thread when the local refcount reaches
69+
// zero. The call will deallocate the object if the shared refcount is also
70+
// zero. Otherwise, the thread gives up ownership and merges the reference
71+
// count fields.
72+
PyAPI_FUNC(void) _Py_MergeZeroLocalRefcount(PyObject *);
73+
#endif
74+
75+
#if defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG))
76+
// Implemented in ../object.h
77+
#elif defined(Py_NOGIL) && defined(Py_REF_DEBUG)
78+
static inline void Py_DECREF(const char *filename, int lineno, PyObject *op)
79+
{
80+
uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local);
81+
if (local == _Py_IMMORTAL_REFCNT_LOCAL) {
82+
return;
83+
}
84+
_Py_DECREF_STAT_INC();
85+
_Py_DECREF_DecRefTotal();
86+
if (_Py_IsOwnedByCurrentThread(op)) {
87+
if (local == 0) {
88+
_Py_NegativeRefcount(filename, lineno, op);
89+
}
90+
local--;
91+
_Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local);
92+
if (local == 0) {
93+
_Py_MergeZeroLocalRefcount(op);
94+
}
95+
}
96+
else {
97+
_Py_DecRefSharedDebug(op, filename, lineno);
98+
}
99+
}
100+
#define Py_DECREF(op) Py_DECREF(__FILE__, __LINE__, _PyObject_CAST(op))
101+
102+
#elif defined(Py_NOGIL)
103+
static inline void Py_DECREF(PyObject *op)
104+
{
105+
uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local);
106+
if (local == _Py_IMMORTAL_REFCNT_LOCAL) {
107+
return;
108+
}
109+
_Py_DECREF_STAT_INC();
110+
if (_Py_IsOwnedByCurrentThread(op)) {
111+
local--;
112+
_Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local);
113+
if (local == 0) {
114+
_Py_MergeZeroLocalRefcount(op);
115+
}
116+
}
117+
else {
118+
_Py_DecRefShared(op);
119+
}
120+
}
121+
#define Py_DECREF(op) Py_DECREF(_PyObject_CAST(op))
122+
123+
#elif defined(Py_REF_DEBUG)
124+
static inline void Py_DECREF(const char *filename, int lineno, PyObject *op)
125+
{
126+
if (op->ob_refcnt <= 0) {
127+
_Py_NegativeRefcount(filename, lineno, op);
128+
}
129+
if (_Py_IsImmortal(op)) {
130+
return;
131+
}
132+
_Py_DECREF_STAT_INC();
133+
_Py_DECREF_DecRefTotal();
134+
if (--op->ob_refcnt == 0) {
135+
_Py_Dealloc(op);
136+
}
137+
}
138+
#define Py_DECREF(op) Py_DECREF(__FILE__, __LINE__, _PyObject_CAST(op))
139+
140+
#else
141+
static inline Py_ALWAYS_INLINE void Py_DECREF(PyObject *op)
142+
{
143+
// Non-limited C API and limited C API for Python 3.9 and older access
144+
// directly PyObject.ob_refcnt.
145+
if (_Py_IsImmortal(op)) {
146+
return;
147+
}
148+
_Py_DECREF_STAT_INC();
149+
if (--op->ob_refcnt == 0) {
150+
_Py_Dealloc(op);
151+
}
152+
}
153+
#define Py_DECREF(op) Py_DECREF(_PyObject_CAST(op))
154+
#endif
155+
156+
5157
PyAPI_FUNC(void) _Py_NewReference(PyObject *op);
6158
PyAPI_FUNC(void) _Py_NewReferenceNoTotal(PyObject *op);
7159

‎Include/object.h

Lines changed: 60 additions & 131 deletions
Original file line numberDiff line numberDiff line change
@@ -717,6 +717,7 @@ PyAPI_FUNC(void) Py_DecRef(PyObject *);
717717
PyAPI_FUNC(void) _Py_IncRef(PyObject *);
718718
PyAPI_FUNC(void) _Py_DecRef(PyObject *);
719719

720+
#if defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG))
720721
static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op)
721722
{
722723
#if defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG))
@@ -765,8 +766,12 @@ static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op)
765766
#endif
766767
#endif
767768
}
768-
#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000
769-
# define Py_INCREF(op) Py_INCREF(_PyObject_CAST(op))
769+
770+
# if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000
771+
# define Py_INCREF(op) Py_INCREF(_PyObject_CAST(op))
772+
# endif
773+
#else
774+
// Implemented in cpython/object.h
770775
#endif
771776

772777

@@ -795,84 +800,8 @@ static inline void Py_DECREF(PyObject *op) {
795800
# endif
796801
}
797802
#define Py_DECREF(op) Py_DECREF(_PyObject_CAST(op))
798-
799-
#elif defined(Py_NOGIL) && defined(Py_REF_DEBUG)
800-
static inline void Py_DECREF(const char *filename, int lineno, PyObject *op)
801-
{
802-
uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local);
803-
if (local == _Py_IMMORTAL_REFCNT_LOCAL) {
804-
return;
805-
}
806-
_Py_DECREF_STAT_INC();
807-
_Py_DECREF_DecRefTotal();
808-
if (_Py_IsOwnedByCurrentThread(op)) {
809-
if (local == 0) {
810-
_Py_NegativeRefcount(filename, lineno, op);
811-
}
812-
local--;
813-
_Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local);
814-
if (local == 0) {
815-
_Py_MergeZeroLocalRefcount(op);
816-
}
817-
}
818-
else {
819-
_Py_DecRefSharedDebug(op, filename, lineno);
820-
}
821-
}
822-
#define Py_DECREF(op) Py_DECREF(__FILE__, __LINE__, _PyObject_CAST(op))
823-
824-
#elif defined(Py_NOGIL)
825-
static inline void Py_DECREF(PyObject *op)
826-
{
827-
uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local);
828-
if (local == _Py_IMMORTAL_REFCNT_LOCAL) {
829-
return;
830-
}
831-
_Py_DECREF_STAT_INC();
832-
if (_Py_IsOwnedByCurrentThread(op)) {
833-
local--;
834-
_Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local);
835-
if (local == 0) {
836-
_Py_MergeZeroLocalRefcount(op);
837-
}
838-
}
839-
else {
840-
_Py_DecRefShared(op);
841-
}
842-
}
843-
#define Py_DECREF(op) Py_DECREF(_PyObject_CAST(op))
844-
845-
#elif defined(Py_REF_DEBUG)
846-
static inline void Py_DECREF(const char *filename, int lineno, PyObject *op)
847-
{
848-
if (op->ob_refcnt <= 0) {
849-
_Py_NegativeRefcount(filename, lineno, op);
850-
}
851-
if (_Py_IsImmortal(op)) {
852-
return;
853-
}
854-
_Py_DECREF_STAT_INC();
855-
_Py_DECREF_DecRefTotal();
856-
if (--op->ob_refcnt == 0) {
857-
_Py_Dealloc(op);
858-
}
859-
}
860-
#define Py_DECREF(op) Py_DECREF(__FILE__, __LINE__, _PyObject_CAST(op))
861-
862803
#else
863-
static inline Py_ALWAYS_INLINE void Py_DECREF(PyObject *op)
864-
{
865-
// Non-limited C API and limited C API for Python 3.9 and older access
866-
// directly PyObject.ob_refcnt.
867-
if (_Py_IsImmortal(op)) {
868-
return;
869-
}
870-
_Py_DECREF_STAT_INC();
871-
if (--op->ob_refcnt == 0) {
872-
_Py_Dealloc(op);
873-
}
874-
}
875-
#define Py_DECREF(op) Py_DECREF(_PyObject_CAST(op))
804+
// Implemented in cpython/object.h
876805
#endif
877806

878807

@@ -948,58 +877,6 @@ static inline Py_ALWAYS_INLINE void Py_DECREF(PyObject *op)
948877
#endif
949878

950879

951-
/* Function to use in case the object pointer can be NULL: */
952-
static inline void Py_XINCREF(PyObject *op)
953-
{
954-
if (op != _Py_NULL) {
955-
Py_INCREF(op);
956-
}
957-
}
958-
#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000
959-
# define Py_XINCREF(op) Py_XINCREF(_PyObject_CAST(op))
960-
#endif
961-
962-
static inline void Py_XDECREF(PyObject *op)
963-
{
964-
if (op != _Py_NULL) {
965-
Py_DECREF(op);
966-
}
967-
}
968-
#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000
969-
# define Py_XDECREF(op) Py_XDECREF(_PyObject_CAST(op))
970-
#endif
971-
972-
// Create a new strong reference to an object:
973-
// increment the reference count of the object and return the object.
974-
PyAPI_FUNC(PyObject*) Py_NewRef(PyObject *obj);
975-
976-
// Similar to Py_NewRef(), but the object can be NULL.
977-
PyAPI_FUNC(PyObject*) Py_XNewRef(PyObject *obj);
978-
979-
static inline PyObject* _Py_NewRef(PyObject *obj)
980-
{
981-
Py_INCREF(obj);
982-
return obj;
983-
}
984-
985-
static inline PyObject* _Py_XNewRef(PyObject *obj)
986-
{
987-
Py_XINCREF(obj);
988-
return obj;
989-
}
990-
991-
// Py_NewRef() and Py_XNewRef() are exported as functions for the stable ABI.
992-
// Names overridden with macros by static inline functions for best
993-
// performances.
994-
#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000
995-
# define Py_NewRef(obj) _Py_NewRef(_PyObject_CAST(obj))
996-
# define Py_XNewRef(obj) _Py_XNewRef(_PyObject_CAST(obj))
997-
#else
998-
# define Py_NewRef(obj) _Py_NewRef(obj)
999-
# define Py_XNewRef(obj) _Py_XNewRef(obj)
1000-
#endif
1001-
1002-
1003880
/*
1004881
_Py_NoneStruct is an object of undefined type which can be used in contexts
1005882
where NULL (nil) is not suitable (since NULL often means 'error').
@@ -1120,6 +997,58 @@ times.
1120997
#endif
1121998

1122999

1000+
/* Function to use in case the object pointer can be NULL: */
1001+
static inline void Py_XINCREF(PyObject *op)
1002+
{
1003+
if (op != _Py_NULL) {
1004+
Py_INCREF(op);
1005+
}
1006+
}
1007+
#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000
1008+
# define Py_XINCREF(op) Py_XINCREF(_PyObject_CAST(op))
1009+
#endif
1010+
1011+
static inline void Py_XDECREF(PyObject *op)
1012+
{
1013+
if (op != _Py_NULL) {
1014+
Py_DECREF(op);
1015+
}
1016+
}
1017+
#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000
1018+
# define Py_XDECREF(op) Py_XDECREF(_PyObject_CAST(op))
1019+
#endif
1020+
1021+
// Create a new strong reference to an object:
1022+
// increment the reference count of the object and return the object.
1023+
PyAPI_FUNC(PyObject*) Py_NewRef(PyObject *obj);
1024+
1025+
// Similar to Py_NewRef(), but the object can be NULL.
1026+
PyAPI_FUNC(PyObject*) Py_XNewRef(PyObject *obj);
1027+
1028+
static inline PyObject* _Py_NewRef(PyObject *obj)
1029+
{
1030+
Py_INCREF(obj);
1031+
return obj;
1032+
}
1033+
1034+
static inline PyObject* _Py_XNewRef(PyObject *obj)
1035+
{
1036+
Py_XINCREF(obj);
1037+
return obj;
1038+
}
1039+
1040+
// Py_NewRef() and Py_XNewRef() are exported as functions for the stable ABI.
1041+
// Names overridden with macros by static inline functions for best
1042+
// performances.
1043+
#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000
1044+
# define Py_NewRef(obj) _Py_NewRef(_PyObject_CAST(obj))
1045+
# define Py_XNewRef(obj) _Py_XNewRef(_PyObject_CAST(obj))
1046+
#else
1047+
# define Py_NewRef(obj) _Py_NewRef(obj)
1048+
# define Py_XNewRef(obj) _Py_XNewRef(obj)
1049+
#endif
1050+
1051+
11231052
static inline int
11241053
PyType_HasFeature(PyTypeObject *type, unsigned long feature)
11251054
{

0 commit comments

Comments
 (0)