Skip to content

Commit 1d6d5e8

Browse files
colesburycolesbury
authored
gh-112529: Use GC heaps for GC allocations in free-threaded builds (gh-114157)
* gh-112529: Use GC heaps for GC allocations in free-threaded builds The free-threaded build's garbage collector implementation will need to find GC objects by traversing mimalloc heaps. This hooks up the allocation calls with the correct heaps by using a thread-local "current_obj_heap" variable. * Refactor out setting heap based on type
1 parent b1ad5a5 commit 1d6d5e8

File tree

6 files changed

+85
-7
lines changed

6 files changed

+85
-7
lines changed

‎Include/internal/pycore_object_alloc.h‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
#ifndef Py_INTERNAL_OBJECT_ALLOC_H
2+
#define Py_INTERNAL_OBJECT_ALLOC_H
3+
4+
#include "pycore_object.h" // _PyType_HasFeature()
5+
#include "pycore_pystate.h" // _PyThreadState_GET()
6+
#include "pycore_tstate.h" // _PyThreadStateImpl
7+
8+
#ifdef __cplusplus
9+
extern "C" {
10+
#endif
11+
12+
#ifndef Py_BUILD_CORE
13+
# error "this header requires Py_BUILD_CORE define"
14+
#endif
15+
16+
#ifdef Py_GIL_DISABLED
17+
static inline mi_heap_t *
18+
_PyObject_GetAllocationHeap(_PyThreadStateImpl *tstate, PyTypeObject *tp)
19+
{
20+
struct _mimalloc_thread_state *m = &tstate->mimalloc;
21+
if (_PyType_HasFeature(tp, Py_TPFLAGS_PREHEADER)) {
22+
return &m->heaps[_Py_MIMALLOC_HEAP_GC_PRE];
23+
}
24+
else if (_PyType_IS_GC(tp)) {
25+
return &m->heaps[_Py_MIMALLOC_HEAP_GC];
26+
}
27+
else {
28+
return &m->heaps[_Py_MIMALLOC_HEAP_OBJECT];
29+
}
30+
}
31+
#endif
32+
33+
// Sets the heap used for PyObject_Malloc(), PyObject_Realloc(), etc. calls in
34+
// Py_GIL_DISABLED builds. We use different heaps depending on if the object
35+
// supports GC and if it has a pre-header. We smuggle the choice of heap
36+
// through the _mimalloc_thread_state. In the default build, this simply
37+
// calls PyObject_Malloc().
38+
static inline void *
39+
_PyObject_MallocWithType(PyTypeObject *tp, size_t size)
40+
{
41+
#ifdef Py_GIL_DISABLED
42+
_PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET();
43+
struct _mimalloc_thread_state *m = &tstate->mimalloc;
44+
m->current_object_heap = _PyObject_GetAllocationHeap(tstate, tp);
45+
#endif
46+
void *mem = PyObject_Malloc(size);
47+
#ifdef Py_GIL_DISABLED
48+
m->current_object_heap = &m->heaps[_Py_MIMALLOC_HEAP_OBJECT];
49+
#endif
50+
return mem;
51+
}
52+
53+
static inline void *
54+
_PyObject_ReallocWithType(PyTypeObject *tp, void *ptr, size_t size)
55+
{
56+
#ifdef Py_GIL_DISABLED
57+
_PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET();
58+
struct _mimalloc_thread_state *m = &tstate->mimalloc;
59+
m->current_object_heap = _PyObject_GetAllocationHeap(tstate, tp);
60+
#endif
61+
void *mem = PyObject_Realloc(ptr, size);
62+
#ifdef Py_GIL_DISABLED
63+
m->current_object_heap = &m->heaps[_Py_MIMALLOC_HEAP_OBJECT];
64+
#endif
65+
return mem;
66+
}
67+
68+
#ifdef __cplusplus
69+
}
70+
#endif
71+
#endif // !Py_INTERNAL_OBJECT_ALLOC_H

‎Makefile.pre.in‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1852,6 +1852,7 @@ PYTHON_HEADERS= \
18521852
$(srcdir)/Include/internal/pycore_moduleobject.h \
18531853
$(srcdir)/Include/internal/pycore_namespace.h \
18541854
$(srcdir)/Include/internal/pycore_object.h \
1855+
$(srcdir)/Include/internal/pycore_object_alloc.h \
18551856
$(srcdir)/Include/internal/pycore_object_state.h \
18561857
$(srcdir)/Include/internal/pycore_obmalloc.h \
18571858
$(srcdir)/Include/internal/pycore_obmalloc_init.h \

‎Objects/typeobject.c‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
#include "pycore_modsupport.h" // _PyArg_NoKwnames()
1212
#include "pycore_moduleobject.h" // _PyModule_GetDef()
1313
#include "pycore_object.h" // _PyType_HasFeature()
14+
#include "pycore_object_alloc.h" // _PyObject_MallocWithType()
1415
#include "pycore_pyerrors.h" // _PyErr_Occurred()
1516
#include "pycore_pystate.h" // _PyThreadState_GET()
1617
#include "pycore_symtable.h" // _Py_Mangle()
@@ -1729,7 +1730,7 @@ _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems)
17291730
const size_t size = _PyObject_VAR_SIZE(type, nitems+1);
17301731

17311732
const size_t presize = _PyType_PreHeaderSize(type);
1732-
char *alloc = PyObject_Malloc(size + presize);
1733+
char *alloc = _PyObject_MallocWithType(type, size + presize);
17331734
if (alloc == NULL) {
17341735
return PyErr_NoMemory();
17351736
}

‎PCbuild/pythoncore.vcxproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -255,6 +255,7 @@
255255
<ClInclude Include="..\Include\internal\pycore_moduleobject.h" />
256256
<ClInclude Include="..\Include\internal\pycore_namespace.h" />
257257
<ClInclude Include="..\Include\internal\pycore_object.h" />
258+
<ClInclude Include="..\Include\internal\pycore_object_alloc.h" />
258259
<ClInclude Include="..\Include\internal\pycore_object_state.h" />
259260
<ClInclude Include="..\Include\internal\pycore_obmalloc.h" />
260261
<ClInclude Include="..\Include\internal\pycore_obmalloc_init.h" />

‎PCbuild/pythoncore.vcxproj.filters‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -693,6 +693,9 @@
693693
<ClInclude Include="..\Include\internal\pycore_object.h">
694694
<Filter>Include\internal</Filter>
695695
</ClInclude>
696+
<ClInclude Include="..\Include\internal\pycore_object_alloc.h">
697+
<Filter>Include\internal</Filter>
698+
</ClInclude>
696699
<ClInclude Include="..\Include\internal\pycore_object_state.h">
697700
<Filter>Include\internal</Filter>
698701
</ClInclude>

‎Python/gc.c‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
#include "pycore_initconfig.h"
1010
#include "pycore_interp.h" // PyInterpreterState.gc
1111
#include "pycore_object.h"
12+
#include "pycore_object_alloc.h" // _PyObject_MallocWithType()
1213
#include "pycore_pyerrors.h"
1314
#include "pycore_pystate.h" // _PyThreadState_GET()
1415
#include "pycore_weakref.h" // _PyWeakref_ClearRef()
@@ -1795,14 +1796,14 @@ _Py_RunGC(PyThreadState *tstate)
17951796
}
17961797

17971798
static PyObject *
1798-
gc_alloc(size_t basicsize, size_t presize)
1799+
gc_alloc(PyTypeObject *tp, size_t basicsize, size_t presize)
17991800
{
18001801
PyThreadState *tstate = _PyThreadState_GET();
18011802
if (basicsize > PY_SSIZE_T_MAX - presize) {
18021803
return _PyErr_NoMemory(tstate);
18031804
}
18041805
size_t size = presize + basicsize;
1805-
char *mem = PyObject_Malloc(size);
1806+
char *mem = _PyObject_MallocWithType(tp, size);
18061807
if (mem == NULL) {
18071808
return _PyErr_NoMemory(tstate);
18081809
}
@@ -1817,7 +1818,7 @@ PyObject *
18171818
_PyObject_GC_New(PyTypeObject *tp)
18181819
{
18191820
size_t presize = _PyType_PreHeaderSize(tp);
1820-
PyObject *op = gc_alloc(_PyObject_SIZE(tp), presize);
1821+
PyObject *op = gc_alloc(tp, _PyObject_SIZE(tp), presize);
18211822
if (op == NULL) {
18221823
return NULL;
18231824
}
@@ -1836,7 +1837,7 @@ _PyObject_GC_NewVar(PyTypeObject *tp, Py_ssize_t nitems)
18361837
}
18371838
size_t presize = _PyType_PreHeaderSize(tp);
18381839
size_t size = _PyObject_VAR_SIZE(tp, nitems);
1839-
op = (PyVarObject *)gc_alloc(size, presize);
1840+
op = (PyVarObject *)gc_alloc(tp, size, presize);
18401841
if (op == NULL) {
18411842
return NULL;
18421843
}
@@ -1848,7 +1849,7 @@ PyObject *
18481849
PyUnstable_Object_GC_NewWithExtraData(PyTypeObject *tp, size_t extra_size)
18491850
{
18501851
size_t presize = _PyType_PreHeaderSize(tp);
1851-
PyObject *op = gc_alloc(_PyObject_SIZE(tp) + extra_size, presize);
1852+
PyObject *op = gc_alloc(tp, _PyObject_SIZE(tp) + extra_size, presize);
18521853
if (op == NULL) {
18531854
return NULL;
18541855
}
@@ -1867,7 +1868,7 @@ _PyObject_GC_Resize(PyVarObject *op, Py_ssize_t nitems)
18671868
return (PyVarObject *)PyErr_NoMemory();
18681869
}
18691870
char *mem = (char *)op - presize;
1870-
mem = (char *)PyObject_Realloc(mem, presize + basicsize);
1871+
mem = (char *)_PyObject_ReallocWithType(Py_TYPE(op), mem, presize + basicsize);
18711872
if (mem == NULL) {
18721873
return (PyVarObject *)PyErr_NoMemory();
18731874
}

0 commit comments

Comments
 (0)