Skip to content

Commit d764284

Browse files
committed
Free objects with qsbr if shared
1 parent 88b5c66 commit d764284

File tree

3 files changed

+69
-10
lines changed

3 files changed

+69
-10
lines changed

Include/internal/pycore_pymem.h

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,13 @@ extern int _PyMem_DebugEnabled(void);
119119
// Enqueue a pointer to be freed possibly after some delay.
120120
extern void _PyMem_FreeDelayed(void *ptr);
121121

122+
// Enqueue an object to be freed possibly after some delay
123+
extern void _PyObject_FreeDelayed(void *ptr);
124+
125+
// Free all outstanding delayed freed objects. The world must be
126+
// stopped before calling this.
127+
extern void _PyMem_ProcessAllDelayed(PyInterpreterState *interp);
128+
122129
// Periodically process delayed free requests.
123130
extern void _PyMem_ProcessDelayed(PyThreadState *tstate);
124131

Objects/obmalloc.c

Lines changed: 50 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -957,7 +957,7 @@ _PyMem_Strdup(const char *str)
957957

958958
// A pointer to be freed once the QSBR read sequence reaches qsbr_goal.
959959
struct _mem_work_item {
960-
void *ptr;
960+
void *ptr; // lowest bit tagged 1 for objects freed with PyObject_Free
961961
uint64_t qsbr_goal;
962962
};
963963

@@ -971,16 +971,27 @@ struct _mem_work_chunk {
971971
struct _mem_work_item array[WORK_ITEMS_PER_CHUNK];
972972
};
973973

974-
void
975-
_PyMem_FreeDelayed(void *ptr)
974+
static void
975+
free_work_item(void *ptr)
976+
{
977+
if (((uintptr_t)ptr) & 0x01) {
978+
PyObject_Free(((char *)ptr) - 1);
979+
}
980+
else {
981+
PyMem_Free(ptr);
982+
}
983+
}
984+
985+
static void
986+
free_delayed(void *ptr)
976987
{
977988
#ifndef Py_GIL_DISABLED
978-
PyMem_Free(ptr);
989+
free_work_item(ptr);
979990
#else
980991
if (_PyRuntime.stoptheworld.world_stopped) {
981992
// Free immediately if the world is stopped, including during
982993
// interpreter shutdown.
983-
PyMem_Free(ptr);
994+
free_work_item(ptr);
984995
return;
985996
}
986997

@@ -1007,7 +1018,7 @@ _PyMem_FreeDelayed(void *ptr)
10071018
if (buf == NULL) {
10081019
// failed to allocate a buffer, free immediately
10091020
_PyEval_StopTheWorld(tstate->base.interp);
1010-
PyMem_Free(ptr);
1021+
free_work_item(ptr);
10111022
_PyEval_StartTheWorld(tstate->base.interp);
10121023
return;
10131024
}
@@ -1024,6 +1035,20 @@ _PyMem_FreeDelayed(void *ptr)
10241035
#endif
10251036
}
10261037

1038+
void
1039+
_PyMem_FreeDelayed(void *ptr)
1040+
{
1041+
assert(!((uintptr_t)ptr & 0x01));
1042+
free_delayed(ptr);
1043+
}
1044+
1045+
1046+
void _PyObject_FreeDelayed(void *ptr)
1047+
{
1048+
assert(!((uintptr_t)ptr & 0x01));
1049+
free_delayed((void *)(((uintptr_t)ptr)|0x01));
1050+
}
1051+
10271052
static struct _mem_work_chunk *
10281053
work_queue_first(struct llist_node *head)
10291054
{
@@ -1043,7 +1068,7 @@ process_queue(struct llist_node *head, struct _qsbr_thread_state *qsbr,
10431068
return;
10441069
}
10451070

1046-
PyMem_Free(item->ptr);
1071+
free_work_item(item->ptr);
10471072
buf->rd_idx++;
10481073
}
10491074

@@ -1119,6 +1144,23 @@ _PyMem_AbandonDelayed(PyThreadState *tstate)
11191144
assert(llist_empty(queue)); // the thread's queue is now empty
11201145
}
11211146

1147+
void
1148+
_PyMem_ProcessAllDelayed(PyInterpreterState *interp)
1149+
{
1150+
PyThreadState *tstate = PyInterpreterState_ThreadHead(interp);
1151+
while (tstate != NULL) {
1152+
_PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate;
1153+
1154+
// Process thread-local work
1155+
process_queue(&tstate_impl->mem_free_queue, tstate_impl->qsbr, true);
1156+
1157+
tstate = PyThreadState_Next(tstate);
1158+
}
1159+
1160+
// Process shared interpreter work
1161+
process_interp_queue(&interp->mem_free_queue, ((_PyThreadStateImpl *)_PyThreadState_GET())->qsbr);
1162+
}
1163+
11221164
void
11231165
_PyMem_FiniDelayed(PyInterpreterState *interp)
11241166
{
@@ -1130,7 +1172,7 @@ _PyMem_FiniDelayed(PyInterpreterState *interp)
11301172
// Free the remaining items immediately. There should be no other
11311173
// threads accessing the memory at this point during shutdown.
11321174
struct _mem_work_item *item = &buf->array[buf->rd_idx];
1133-
PyMem_Free(item->ptr);
1175+
free_work_item(item->ptr);
11341176
buf->rd_idx++;
11351177
}
11361178

Python/gc_free_threading.c

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1025,6 +1025,11 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state)
10251025
merge_all_queued_objects(interp, state);
10261026
process_delayed_frees(interp);
10271027

1028+
// Free all outstanding objects. During finalization we could collect
1029+
// types before the underlying objects so its important that we not
1030+
// traverse any delay-freed objects.
1031+
_PyMem_ProcessAllDelayed(interp);
1032+
10281033
// Find unreachable objects
10291034
int err = deduce_unreachable_heap(interp, state);
10301035
if (err < 0) {
@@ -1704,8 +1709,13 @@ PyObject_GC_Del(void *op)
17041709
}
17051710

17061711
record_deallocation(_PyThreadState_GET());
1707-
1708-
PyObject_Free(((char *)op)-presize);
1712+
PyObject *self = (PyObject *)op;
1713+
if (_PyObject_GC_IS_SHARED(self)) {
1714+
_PyObject_FreeDelayed(((char *)op)-presize);
1715+
}
1716+
else {
1717+
PyObject_Free(((char *)op)-presize);
1718+
}
17091719
}
17101720

17111721
int

0 commit comments

Comments
 (0)