Skip to content

Commit 1390796

Browse files
corona10corona10
authored
gh-111968: Use per-thread freelists for dict in free-threading (gh-114323)
1 parent 587d480 commit 1390796

13 files changed

+75
-75
lines changed

‎Include/internal/pycore_dict.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ extern "C" {
99
# error "this header requires Py_BUILD_CORE define"
1010
#endif
1111

12+
#include "pycore_freelist.h" // _PyFreeListState
1213
#include "pycore_identifier.h" // _Py_Identifier
1314
#include "pycore_object.h" // PyDictOrValues
1415

@@ -69,7 +70,7 @@ extern PyObject* _PyDictView_Intersect(PyObject* self, PyObject *other);
6970

7071
/* runtime lifecycle */
7172

72-
extern void _PyDict_Fini(PyInterpreterState *interp);
73+
extern void _PyDict_Fini(PyInterpreterState *state);
7374

7475

7576
/* other API */

‎Include/internal/pycore_dict_state.h

Lines changed: 0 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -8,16 +8,6 @@ extern "C" {
88
# error "this header requires Py_BUILD_CORE define"
99
#endif
1010

11-
12-
#ifndef WITH_FREELISTS
13-
// without freelists
14-
# define PyDict_MAXFREELIST 0
15-
#endif
16-
17-
#ifndef PyDict_MAXFREELIST
18-
# define PyDict_MAXFREELIST 80
19-
#endif
20-
2111
#define DICT_MAX_WATCHERS 8
2212

2313
struct _Py_dict_state {
@@ -26,15 +16,6 @@ struct _Py_dict_state {
2616
* time that a dictionary is modified. */
2717
uint64_t global_version;
2818
uint32_t next_keys_version;
29-
30-
#if PyDict_MAXFREELIST > 0
31-
/* Dictionary reuse scheme to save calls to malloc and free */
32-
PyDictObject *free_list[PyDict_MAXFREELIST];
33-
PyDictKeysObject *keys_free_list[PyDict_MAXFREELIST];
34-
int numfree;
35-
int keys_numfree;
36-
#endif
37-
3819
PyDict_WatchCallback watchers[DICT_MAX_WATCHERS];
3920
};
4021

‎Include/internal/pycore_freelist.h

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ extern "C" {
1717
# define PyTuple_NFREELISTS PyTuple_MAXSAVESIZE
1818
# define PyTuple_MAXFREELIST 2000
1919
# define PyList_MAXFREELIST 80
20+
# define PyDict_MAXFREELIST 80
2021
# define PyFloat_MAXFREELIST 100
2122
# define PyContext_MAXFREELIST 255
2223
# define _PyAsyncGen_MAXFREELIST 80
@@ -25,6 +26,7 @@ extern "C" {
2526
# define PyTuple_NFREELISTS 0
2627
# define PyTuple_MAXFREELIST 0
2728
# define PyList_MAXFREELIST 0
29+
# define PyDict_MAXFREELIST 0
2830
# define PyFloat_MAXFREELIST 0
2931
# define PyContext_MAXFREELIST 0
3032
# define _PyAsyncGen_MAXFREELIST 0
@@ -65,6 +67,16 @@ struct _Py_float_state {
6567
#endif
6668
};
6769

70+
struct _Py_dict_freelist {
71+
#ifdef WITH_FREELISTS
72+
/* Dictionary reuse scheme to save calls to malloc and free */
73+
PyDictObject *free_list[PyDict_MAXFREELIST];
74+
PyDictKeysObject *keys_free_list[PyDict_MAXFREELIST];
75+
int numfree;
76+
int keys_numfree;
77+
#endif
78+
};
79+
6880
struct _Py_slice_state {
6981
#ifdef WITH_FREELISTS
7082
/* Using a cache is very effective since typically only a single slice is
@@ -106,6 +118,7 @@ typedef struct _Py_freelist_state {
106118
struct _Py_float_state floats;
107119
struct _Py_tuple_state tuples;
108120
struct _Py_list_state lists;
121+
struct _Py_dict_freelist dicts;
109122
struct _Py_slice_state slices;
110123
struct _Py_context_state contexts;
111124
struct _Py_async_gen_state async_gens;

‎Include/internal/pycore_gc.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -267,7 +267,7 @@ extern void _PyTuple_ClearFreeList(_PyFreeListState *state, int is_finalization)
267267
extern void _PyFloat_ClearFreeList(_PyFreeListState *state, int is_finalization);
268268
extern void _PyList_ClearFreeList(_PyFreeListState *state, int is_finalization);
269269
extern void _PySlice_ClearCache(_PyFreeListState *state);
270-
extern void _PyDict_ClearFreeList(PyInterpreterState *interp);
270+
extern void _PyDict_ClearFreeList(_PyFreeListState *state, int is_finalization);
271271
extern void _PyAsyncGen_ClearFreeLists(_PyFreeListState *state, int is_finalization);
272272
extern void _PyContext_ClearFreeList(_PyFreeListState *state, int is_finalization);
273273
extern void _Py_ScheduleGC(PyInterpreterState *interp);

‎Include/internal/pycore_interp.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ extern "C" {
2020
#include "pycore_dtoa.h" // struct _dtoa_state
2121
#include "pycore_exceptions.h" // struct _Py_exc_state
2222
#include "pycore_floatobject.h" // struct _Py_float_state
23+
#include "pycore_freelist.h" // struct _Py_freelist_state
2324
#include "pycore_function.h" // FUNC_MAX_WATCHERS
2425
#include "pycore_gc.h" // struct _gc_runtime_state
2526
#include "pycore_genobject.h" // struct _Py_async_gen_state
@@ -230,7 +231,6 @@ struct _is {
230231
struct _dtoa_state dtoa;
231232
struct _py_func_state func_state;
232233

233-
struct _Py_tuple_state tuple;
234234
struct _Py_dict_state dict_state;
235235
struct _Py_exc_state exc_state;
236236

‎Objects/dictobject.c

Lines changed: 39 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,7 @@ As a consequence of this, split keys have a maximum size of 16.
118118
#include "pycore_ceval.h" // _PyEval_GetBuiltin()
119119
#include "pycore_code.h" // stats
120120
#include "pycore_dict.h" // export _PyDict_SizeOf()
121+
#include "pycore_freelist.h" // _PyFreeListState_GET()
121122
#include "pycore_gc.h" // _PyObject_GC_IS_TRACKED()
122123
#include "pycore_object.h" // _PyObject_GC_TRACK(), _PyDebugAllocatorStats()
123124
#include "pycore_pyerrors.h" // _PyErr_GetRaisedException()
@@ -242,40 +243,44 @@ static PyObject* dict_iter(PyObject *dict);
242243
#include "clinic/dictobject.c.h"
243244

244245

245-
#if PyDict_MAXFREELIST > 0
246-
static struct _Py_dict_state *
247-
get_dict_state(PyInterpreterState *interp)
246+
#ifdef WITH_FREELISTS
247+
static struct _Py_dict_freelist *
248+
get_dict_state(void)
248249
{
249-
return &interp->dict_state;
250+
_PyFreeListState *state = _PyFreeListState_GET();
251+
return &state->dicts;
250252
}
251253
#endif
252254

253255

254256
void
255-
_PyDict_ClearFreeList(PyInterpreterState *interp)
257+
_PyDict_ClearFreeList(_PyFreeListState *freelist_state, int is_finalization)
256258
{
257-
#if PyDict_MAXFREELIST > 0
258-
struct _Py_dict_state *state = &interp->dict_state;
259-
while (state->numfree) {
259+
#ifdef WITH_FREELISTS
260+
struct _Py_dict_freelist *state = &freelist_state->dicts;
261+
while (state->numfree > 0) {
260262
PyDictObject *op = state->free_list[--state->numfree];
261263
assert(PyDict_CheckExact(op));
262264
PyObject_GC_Del(op);
263265
}
264-
while (state->keys_numfree) {
266+
while (state->keys_numfree > 0) {
265267
PyMem_Free(state->keys_free_list[--state->keys_numfree]);
266268
}
269+
if (is_finalization) {
270+
state->numfree = -1;
271+
state->keys_numfree = -1;
272+
}
267273
#endif
268274
}
269275

270-
271276
void
272-
_PyDict_Fini(PyInterpreterState *interp)
277+
_PyDict_Fini(PyInterpreterState *Py_UNUSED(interp))
273278
{
274-
_PyDict_ClearFreeList(interp);
275-
#if defined(Py_DEBUG) && PyDict_MAXFREELIST > 0
276-
struct _Py_dict_state *state = &interp->dict_state;
277-
state->numfree = -1;
278-
state->keys_numfree = -1;
279+
// With Py_GIL_DISABLED:
280+
// the freelists for the current thread state have already been cleared.
281+
#ifndef Py_GIL_DISABLED
282+
_PyFreeListState *state = _PyFreeListState_GET();
283+
_PyDict_ClearFreeList(state, 1);
279284
#endif
280285
}
281286

@@ -290,17 +295,16 @@ unicode_get_hash(PyObject *o)
290295
void
291296
_PyDict_DebugMallocStats(FILE *out)
292297
{
293-
#if PyDict_MAXFREELIST > 0
294-
PyInterpreterState *interp = _PyInterpreterState_GET();
295-
struct _Py_dict_state *state = get_dict_state(interp);
298+
#ifdef WITH_FREELISTS
299+
struct _Py_dict_freelist *state = get_dict_state();
296300
_PyDebugAllocatorStats(out, "free PyDictObject",
297301
state->numfree, sizeof(PyDictObject));
298302
#endif
299303
}
300304

301305
#define DK_MASK(dk) (DK_SIZE(dk)-1)
302306

303-
static void free_keys_object(PyInterpreterState *interp, PyDictKeysObject *keys);
307+
static void free_keys_object(PyDictKeysObject *keys);
304308

305309
/* PyDictKeysObject has refcounts like PyObject does, so we have the
306310
following two functions to mirror what Py_INCREF() and Py_DECREF() do.
@@ -348,7 +352,7 @@ dictkeys_decref(PyInterpreterState *interp, PyDictKeysObject *dk)
348352
Py_XDECREF(entries[i].me_value);
349353
}
350354
}
351-
free_keys_object(interp, dk);
355+
free_keys_object(dk);
352356
}
353357
}
354358

@@ -643,12 +647,8 @@ new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode)
643647
log2_bytes = log2_size + 2;
644648
}
645649

646-
#if PyDict_MAXFREELIST > 0
647-
struct _Py_dict_state *state = get_dict_state(interp);
648-
#ifdef Py_DEBUG
649-
// new_keys_object() must not be called after _PyDict_Fini()
650-
assert(state->keys_numfree != -1);
651-
#endif
650+
#ifdef WITH_FREELISTS
651+
struct _Py_dict_freelist *state = get_dict_state();
652652
if (log2_size == PyDict_LOG_MINSIZE && unicode && state->keys_numfree > 0) {
653653
dk = state->keys_free_list[--state->keys_numfree];
654654
OBJECT_STAT_INC(from_freelist);
@@ -680,16 +680,13 @@ new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode)
680680
}
681681

682682
static void
683-
free_keys_object(PyInterpreterState *interp, PyDictKeysObject *keys)
683+
free_keys_object(PyDictKeysObject *keys)
684684
{
685-
#if PyDict_MAXFREELIST > 0
686-
struct _Py_dict_state *state = get_dict_state(interp);
687-
#ifdef Py_DEBUG
688-
// free_keys_object() must not be called after _PyDict_Fini()
689-
assert(state->keys_numfree != -1);
690-
#endif
685+
#ifdef WITH_FREELISTS
686+
struct _Py_dict_freelist *state = get_dict_state();
691687
if (DK_LOG_SIZE(keys) == PyDict_LOG_MINSIZE
692688
&& state->keys_numfree < PyDict_MAXFREELIST
689+
&& state->keys_numfree >= 0
693690
&& DK_IS_UNICODE(keys)) {
694691
state->keys_free_list[state->keys_numfree++] = keys;
695692
OBJECT_STAT_INC(to_freelist);
@@ -730,13 +727,9 @@ new_dict(PyInterpreterState *interp,
730727
{
731728
PyDictObject *mp;
732729
assert(keys != NULL);
733-
#if PyDict_MAXFREELIST > 0
734-
struct _Py_dict_state *state = get_dict_state(interp);
735-
#ifdef Py_DEBUG
736-
// new_dict() must not be called after _PyDict_Fini()
737-
assert(state->numfree != -1);
738-
#endif
739-
if (state->numfree) {
730+
#ifdef WITH_FREELISTS
731+
struct _Py_dict_freelist *state = get_dict_state();
732+
if (state->numfree > 0) {
740733
mp = state->free_list[--state->numfree];
741734
assert (mp != NULL);
742735
assert (Py_IS_TYPE(mp, &PyDict_Type));
@@ -1547,7 +1540,7 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp,
15471540
#endif
15481541
assert(oldkeys->dk_kind != DICT_KEYS_SPLIT);
15491542
assert(oldkeys->dk_refcnt == 1);
1550-
free_keys_object(interp, oldkeys);
1543+
free_keys_object(oldkeys);
15511544
}
15521545
}
15531546

@@ -2458,13 +2451,10 @@ dict_dealloc(PyObject *self)
24582451
assert(keys->dk_refcnt == 1 || keys == Py_EMPTY_KEYS);
24592452
dictkeys_decref(interp, keys);
24602453
}
2461-
#if PyDict_MAXFREELIST > 0
2462-
struct _Py_dict_state *state = get_dict_state(interp);
2463-
#ifdef Py_DEBUG
2464-
// new_dict() must not be called after _PyDict_Fini()
2465-
assert(state->numfree != -1);
2466-
#endif
2467-
if (state->numfree < PyDict_MAXFREELIST && Py_IS_TYPE(mp, &PyDict_Type)) {
2454+
#ifdef WITH_FREELISTS
2455+
struct _Py_dict_freelist *state = get_dict_state();
2456+
if (state->numfree < PyDict_MAXFREELIST && state->numfree >=0 &&
2457+
Py_IS_TYPE(mp, &PyDict_Type)) {
24682458
state->free_list[state->numfree++] = mp;
24692459
OBJECT_STAT_INC(to_freelist);
24702460
}

‎Objects/floatobject.c

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2013,7 +2013,11 @@ _PyFloat_ClearFreeList(_PyFreeListState *freelist_state, int is_finalization)
20132013
void
20142014
_PyFloat_Fini(_PyFreeListState *state)
20152015
{
2016+
// With Py_GIL_DISABLED:
2017+
// the freelists for the current thread state have already been cleared.
2018+
#ifndef Py_GIL_DISABLED
20162019
_PyFloat_ClearFreeList(state, 1);
2020+
#endif
20172021
}
20182022

20192023
void

‎Objects/genobject.c

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1685,7 +1685,11 @@ _PyAsyncGen_ClearFreeLists(_PyFreeListState *freelist_state, int is_finalization
16851685
void
16861686
_PyAsyncGen_Fini(_PyFreeListState *state)
16871687
{
1688+
// With Py_GIL_DISABLED:
1689+
// the freelists for the current thread state have already been cleared.
1690+
#ifndef Py_GIL_DISABLED
16881691
_PyAsyncGen_ClearFreeLists(state, 1);
1692+
#endif
16891693
}
16901694

16911695

‎Objects/listobject.c

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,7 +138,11 @@ _PyList_ClearFreeList(_PyFreeListState *freelist_state, int is_finalization)
138138
void
139139
_PyList_Fini(_PyFreeListState *state)
140140
{
141+
// With Py_GIL_DISABLED:
142+
// the freelists for the current thread state have already been cleared.
143+
#ifndef Py_GIL_DISABLED
141144
_PyList_ClearFreeList(state, 1);
145+
#endif
142146
}
143147

144148
/* Print summary info about the state of the optimized allocator */

‎Python/context.c

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1287,7 +1287,11 @@ _PyContext_ClearFreeList(_PyFreeListState *freelist_state, int is_finalization)
12871287
void
12881288
_PyContext_Fini(_PyFreeListState *state)
12891289
{
1290+
// With Py_GIL_DISABLED:
1291+
// the freelists for the current thread state have already been cleared.
1292+
#ifndef Py_GIL_DISABLED
12901293
_PyContext_ClearFreeList(state, 1);
1294+
#endif
12911295
}
12921296

12931297

‎Python/gc_free_threading.c

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1676,8 +1676,6 @@ PyUnstable_GC_VisitObjects(gcvisitobjects_t callback, void *arg)
16761676
void
16771677
_PyGC_ClearAllFreeLists(PyInterpreterState *interp)
16781678
{
1679-
_PyDict_ClearFreeList(interp);
1680-
16811679
HEAD_LOCK(&_PyRuntime);
16821680
_PyThreadStateImpl *tstate = (_PyThreadStateImpl *)interp->threads.head;
16831681
while (tstate != NULL) {

‎Python/gc_gil.c

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,6 @@
1111
void
1212
_PyGC_ClearAllFreeLists(PyInterpreterState *interp)
1313
{
14-
_PyDict_ClearFreeList(interp);
15-
1614
_Py_ClearFreeLists(&interp->freelist_state, 0);
1715
}
1816

‎Python/pystate.c

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1461,9 +1461,12 @@ clear_datastack(PyThreadState *tstate)
14611461
void
14621462
_Py_ClearFreeLists(_PyFreeListState *state, int is_finalization)
14631463
{
1464+
// In the free-threaded build, freelists are per-PyThreadState and cleared in PyThreadState_Clear()
1465+
// In the default build, freelists are per-interpreter and cleared in finalize_interp_types()
14641466
_PyFloat_ClearFreeList(state, is_finalization);
14651467
_PyTuple_ClearFreeList(state, is_finalization);
14661468
_PyList_ClearFreeList(state, is_finalization);
1469+
_PyDict_ClearFreeList(state, is_finalization);
14671470
_PyContext_ClearFreeList(state, is_finalization);
14681471
_PyAsyncGen_ClearFreeLists(state, is_finalization);
14691472
_PyObjectStackChunk_ClearFreeList(state, is_finalization);

0 commit comments

Comments
 (0)