Skip to content

Commit ada319b

Browse files
pitroupitrou
authored
bpo-32388: Remove cross-version binary compatibility requirement in tp_flags (GH-4944)
It is now allowed to add new fields at the end of the PyTypeObject struct without having to allocate a dedicated compatibility flag in tp_flags. This will reduce the risk of running out of bits in the 32-bit tp_flags value.
1 parent 43fdbd2 commit ada319b

File tree

18 files changed

+52
-51
lines changed

18 files changed

+52
-51
lines changed

‎Doc/c-api/typeobj.rst

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1099,6 +1099,11 @@ and :c:type:`PyType_Type` effectively act as defaults.)
10991099

11001100
.. versionadded:: 3.4
11011101

1102+
.. deprecated:: 3.8
1103+
This flag isn't necessary anymore, as the interpreter assumes the
1104+
:c:member:`~PyTypeObject.tp_finalize` slot is always present in the
1105+
type structure.
1106+
11021107

11031108
.. c:member:: const char* PyTypeObject.tp_doc
11041109

‎Doc/whatsnew/3.8.rst

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1265,6 +1265,15 @@ Changes in the C API
12651265

12661266
(Contributed by Zackery Spytz in :issue:`33407`.)
12671267

1268+
* The interpreter does not pretend to support binary compatibility of
1269+
extension types accross feature releases, anymore. A :c:type:`PyTypeObject`
1270+
exported by a third-party extension module is supposed to have all the
1271+
slots expected in the current Python version, including
1272+
:c:member:`~PyTypeObject.tp_finalize` (:const:`Py_TPFLAGS_HAVE_FINALIZE`
1273+
is not checked anymore before reading :c:member:`~PyTypeObject.tp_finalize`).
1274+
1275+
(Contributed by Antoine Pitrou in :issue:`32388`.)
1276+
12681277

12691278
CPython bytecode changes
12701279
------------------------

‎Include/object.h

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -263,17 +263,14 @@ PyAPI_FUNC(void) Py_ReprLeave(PyObject *);
263263
#define Py_PRINT_RAW 1 /* No string quotes etc. */
264264

265265
/*
266-
`Type flags (tp_flags)
266+
Type flags (tp_flags)
267267
268-
These flags are used to extend the type structure in a backwards-compatible
269-
fashion. Extensions can use the flags to indicate (and test) when a given
270-
type structure contains a new feature. The Python core will use these when
271-
introducing new functionality between major revisions (to avoid mid-version
272-
changes in the PYTHON_API_VERSION).
268+
These flags are used to change expected features and behavior for a
269+
particular type.
273270
274271
Arbitration of the flag bit positions will need to be coordinated among
275272
all extension writers who publicly release their extensions (this will
276-
be fewer than you might expect!)..
273+
be fewer than you might expect!).
277274
278275
Most flags were removed as of Python 3.0 to make room for new flags. (Some
279276
flags are not for backwards compatibility but to indicate the presence of an
@@ -302,7 +299,7 @@ given type object has a specified feature.
302299
/* Set while the type is being 'readied', to prevent recursive ready calls */
303300
#define Py_TPFLAGS_READYING (1UL << 13)
304301

305-
/* Objects support garbage collection (see objimp.h) */
302+
/* Objects support garbage collection (see objimpl.h) */
306303
#define Py_TPFLAGS_HAVE_GC (1UL << 14)
307304

308305
/* These two bits are preserved for Stackless Python, next after this is 17 */
@@ -340,6 +337,11 @@ given type object has a specified feature.
340337
/* NOTE: The following flags reuse lower bits (removed as part of the
341338
* Python 3.0 transition). */
342339

340+
/* The following flag is kept for compatibility. Starting with 3.8,
341+
* binary compatibility of C extensions accross feature releases of
342+
* Python is not supported anymore, except when using the stable ABI.
343+
*/
344+
343345
/* Type structure has tp_finalize member (3.4) */
344346
#define Py_TPFLAGS_HAVE_FINALIZE (1UL << 0)
345347

‎Misc/NEWS.d/next/Core and Builtins/2017-12-21-20-37-40.bpo-32388.6w-i5t.rst

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Remove cross-version binary compatibility requirement in tp_flags.

‎Modules/_asynciomodule.c

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1430,8 +1430,7 @@ static PyTypeObject FutureType = {
14301430
.tp_dealloc = FutureObj_dealloc,
14311431
.tp_as_async = &FutureType_as_async,
14321432
.tp_repr = (reprfunc)FutureObj_repr,
1433-
.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE
1434-
| Py_TPFLAGS_HAVE_FINALIZE,
1433+
.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE,
14351434
.tp_doc = _asyncio_Future___init____doc__,
14361435
.tp_traverse = (traverseproc)FutureObj_traverse,
14371436
.tp_clear = (inquiry)FutureObj_clear,
@@ -2461,8 +2460,7 @@ static PyTypeObject TaskType = {
24612460
.tp_dealloc = TaskObj_dealloc,
24622461
.tp_as_async = &FutureType_as_async,
24632462
.tp_repr = (reprfunc)FutureObj_repr,
2464-
.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE
2465-
| Py_TPFLAGS_HAVE_FINALIZE,
2463+
.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE,
24662464
.tp_doc = _asyncio_Task___init____doc__,
24672465
.tp_traverse = (traverseproc)TaskObj_traverse,
24682466
.tp_clear = (inquiry)TaskObj_clear,

‎Modules/_io/bufferedio.c

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2342,8 +2342,7 @@ PyTypeObject PyBufferedIOBase_Type = {
23422342
0, /*tp_getattro*/
23432343
0, /*tp_setattro*/
23442344
0, /*tp_as_buffer*/
2345-
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE
2346-
| Py_TPFLAGS_HAVE_FINALIZE, /*tp_flags*/
2345+
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/
23472346
bufferediobase_doc, /* tp_doc */
23482347
0, /* tp_traverse */
23492348
0, /* tp_clear */
@@ -2434,7 +2433,7 @@ PyTypeObject PyBufferedReader_Type = {
24342433
0, /*tp_setattro*/
24352434
0, /*tp_as_buffer*/
24362435
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE
2437-
| Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_FINALIZE, /*tp_flags*/
2436+
| Py_TPFLAGS_HAVE_GC, /*tp_flags*/
24382437
_io_BufferedReader___init____doc__, /* tp_doc */
24392438
(traverseproc)buffered_traverse, /* tp_traverse */
24402439
(inquiry)buffered_clear, /* tp_clear */
@@ -2520,7 +2519,7 @@ PyTypeObject PyBufferedWriter_Type = {
25202519
0, /*tp_setattro*/
25212520
0, /*tp_as_buffer*/
25222521
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE
2523-
| Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_FINALIZE, /*tp_flags*/
2522+
| Py_TPFLAGS_HAVE_GC, /*tp_flags*/
25242523
_io_BufferedWriter___init____doc__, /* tp_doc */
25252524
(traverseproc)buffered_traverse, /* tp_traverse */
25262525
(inquiry)buffered_clear, /* tp_clear */
@@ -2597,7 +2596,7 @@ PyTypeObject PyBufferedRWPair_Type = {
25972596
0, /*tp_setattro*/
25982597
0, /*tp_as_buffer*/
25992598
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE
2600-
| Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_FINALIZE, /* tp_flags */
2599+
| Py_TPFLAGS_HAVE_GC, /* tp_flags */
26012600
_io_BufferedRWPair___init____doc__, /* tp_doc */
26022601
(traverseproc)bufferedrwpair_traverse, /* tp_traverse */
26032602
(inquiry)bufferedrwpair_clear, /* tp_clear */
@@ -2691,7 +2690,7 @@ PyTypeObject PyBufferedRandom_Type = {
26912690
0, /*tp_setattro*/
26922691
0, /*tp_as_buffer*/
26932692
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE
2694-
| Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_FINALIZE, /*tp_flags*/
2693+
| Py_TPFLAGS_HAVE_GC, /*tp_flags*/
26952694
_io_BufferedRandom___init____doc__, /* tp_doc */
26962695
(traverseproc)buffered_traverse, /* tp_traverse */
26972696
(inquiry)buffered_clear, /* tp_clear */

‎Modules/_io/fileio.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1200,12 +1200,12 @@ PyTypeObject PyFileIO_Type = {
12001200
0, /* tp_setattro */
12011201
0, /* tp_as_buffer */
12021202
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE
1203-
| Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_FINALIZE, /* tp_flags */
1203+
| Py_TPFLAGS_HAVE_GC, /* tp_flags */
12041204
_io_FileIO___init____doc__, /* tp_doc */
12051205
(traverseproc)fileio_traverse, /* tp_traverse */
12061206
(inquiry)fileio_clear, /* tp_clear */
12071207
0, /* tp_richcompare */
1208-
offsetof(fileio, weakreflist), /* tp_weaklistoffset */
1208+
offsetof(fileio, weakreflist), /* tp_weaklistoffset */
12091209
0, /* tp_iter */
12101210
0, /* tp_iternext */
12111211
fileio_methods, /* tp_methods */
@@ -1215,7 +1215,7 @@ PyTypeObject PyFileIO_Type = {
12151215
0, /* tp_dict */
12161216
0, /* tp_descr_get */
12171217
0, /* tp_descr_set */
1218-
offsetof(fileio, dict), /* tp_dictoffset */
1218+
offsetof(fileio, dict), /* tp_dictoffset */
12191219
_io_FileIO___init__, /* tp_init */
12201220
PyType_GenericAlloc, /* tp_alloc */
12211221
fileio_new, /* tp_new */

‎Modules/_io/iobase.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -856,7 +856,7 @@ PyTypeObject PyIOBase_Type = {
856856
0, /*tp_setattro*/
857857
0, /*tp_as_buffer*/
858858
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE
859-
| Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_FINALIZE, /*tp_flags*/
859+
| Py_TPFLAGS_HAVE_GC, /*tp_flags*/
860860
iobase_doc, /* tp_doc */
861861
(traverseproc)iobase_traverse, /* tp_traverse */
862862
(inquiry)iobase_clear, /* tp_clear */
@@ -1051,7 +1051,7 @@ PyTypeObject PyRawIOBase_Type = {
10511051
0, /*tp_getattro*/
10521052
0, /*tp_setattro*/
10531053
0, /*tp_as_buffer*/
1054-
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_FINALIZE, /*tp_flags*/
1054+
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/
10551055
rawiobase_doc, /* tp_doc */
10561056
0, /* tp_traverse */
10571057
0, /* tp_clear */

‎Modules/_io/textio.c

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -183,8 +183,7 @@ PyTypeObject PyTextIOBase_Type = {
183183
0, /*tp_getattro*/
184184
0, /*tp_setattro*/
185185
0, /*tp_as_buffer*/
186-
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE
187-
| Py_TPFLAGS_HAVE_FINALIZE, /*tp_flags*/
186+
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/
188187
textiobase_doc, /* tp_doc */
189188
0, /* tp_traverse */
190189
0, /* tp_clear */
@@ -3258,7 +3257,7 @@ PyTypeObject PyTextIOWrapper_Type = {
32583257
0, /*tp_setattro*/
32593258
0, /*tp_as_buffer*/
32603259
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE
3261-
| Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_FINALIZE, /*tp_flags*/
3260+
| Py_TPFLAGS_HAVE_GC, /*tp_flags*/
32623261
_io_TextIOWrapper___init____doc__, /* tp_doc */
32633262
(traverseproc)textiowrapper_traverse, /* tp_traverse */
32643263
(inquiry)textiowrapper_clear, /* tp_clear */

‎Modules/_io/winconsoleio.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1133,7 +1133,7 @@ PyTypeObject PyWindowsConsoleIO_Type = {
11331133
0, /* tp_setattro */
11341134
0, /* tp_as_buffer */
11351135
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE
1136-
| Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_FINALIZE, /* tp_flags */
1136+
| Py_TPFLAGS_HAVE_GC, /* tp_flags */
11371137
_io__WindowsConsoleIO___init____doc__, /* tp_doc */
11381138
(traverseproc)winconsoleio_traverse, /* tp_traverse */
11391139
(inquiry)winconsoleio_clear, /* tp_clear */

‎Modules/_testmultiphase.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ static PyType_Spec Example_Type_spec = {
9898
"_testimportexec.Example",
9999
sizeof(ExampleObject),
100100
0,
101-
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_FINALIZE,
101+
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,
102102
Example_Type_slots
103103
};
104104

‎Modules/gcmodule.c

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -858,7 +858,6 @@ finalize_garbage(PyGC_Head *collectable)
858858
PyObject *op = FROM_GC(gc);
859859
gc_list_move(gc, &seen);
860860
if (!_PyGCHead_FINALIZED(gc) &&
861-
PyType_HasFeature(Py_TYPE(op), Py_TPFLAGS_HAVE_FINALIZE) &&
862861
(finalize = Py_TYPE(op)->tp_finalize) != NULL) {
863862
_PyGCHead_SET_FINALIZED(gc);
864863
Py_INCREF(op);

‎Modules/posixmodule.c

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13032,8 +13032,7 @@ static PyTypeObject ScandirIteratorType = {
1303213032
0, /* tp_getattro */
1303313033
0, /* tp_setattro */
1303413034
0, /* tp_as_buffer */
13035-
Py_TPFLAGS_DEFAULT
13036-
| Py_TPFLAGS_HAVE_FINALIZE, /* tp_flags */
13035+
Py_TPFLAGS_DEFAULT, /* tp_flags */
1303713036
0, /* tp_doc */
1303813037
0, /* tp_traverse */
1303913038
0, /* tp_clear */

‎Modules/socketmodule.c

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5286,8 +5286,7 @@ static PyTypeObject sock_type = {
52865286
PyObject_GenericGetAttr, /* tp_getattro */
52875287
0, /* tp_setattro */
52885288
0, /* tp_as_buffer */
5289-
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE
5290-
| Py_TPFLAGS_HAVE_FINALIZE, /* tp_flags */
5289+
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
52915290
sock_doc, /* tp_doc */
52925291
0, /* tp_traverse */
52935292
0, /* tp_clear */

‎Modules/xxlimited.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,7 @@ static PyType_Spec Xxo_Type_spec = {
123123
"xxlimited.Xxo",
124124
sizeof(XxoObject),
125125
0,
126-
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_FINALIZE,
126+
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,
127127
Xxo_Type_slots
128128
};
129129

‎Objects/genobject.c

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -742,8 +742,7 @@ PyTypeObject PyGen_Type = {
742742
PyObject_GenericGetAttr, /* tp_getattro */
743743
0, /* tp_setattro */
744744
0, /* tp_as_buffer */
745-
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC |
746-
Py_TPFLAGS_HAVE_FINALIZE, /* tp_flags */
745+
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */
747746
0, /* tp_doc */
748747
(traverseproc)gen_traverse, /* tp_traverse */
749748
0, /* tp_clear */
@@ -997,8 +996,7 @@ PyTypeObject PyCoro_Type = {
997996
PyObject_GenericGetAttr, /* tp_getattro */
998997
0, /* tp_setattro */
999998
0, /* tp_as_buffer */
1000-
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC |
1001-
Py_TPFLAGS_HAVE_FINALIZE, /* tp_flags */
999+
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */
10021000
0, /* tp_doc */
10031001
(traverseproc)gen_traverse, /* tp_traverse */
10041002
0, /* tp_clear */
@@ -1394,8 +1392,7 @@ PyTypeObject PyAsyncGen_Type = {
13941392
PyObject_GenericGetAttr, /* tp_getattro */
13951393
0, /* tp_setattro */
13961394
0, /* tp_as_buffer */
1397-
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC |
1398-
Py_TPFLAGS_HAVE_FINALIZE, /* tp_flags */
1395+
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */
13991396
0, /* tp_doc */
14001397
(traverseproc)async_gen_traverse, /* tp_traverse */
14011398
0, /* tp_clear */

‎Objects/object.c

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -298,10 +298,7 @@ PyObject_CallFinalizer(PyObject *self)
298298
{
299299
PyTypeObject *tp = Py_TYPE(self);
300300

301-
/* The former could happen on heaptypes created from the C API, e.g.
302-
PyType_FromSpec(). */
303-
if (!PyType_HasFeature(tp, Py_TPFLAGS_HAVE_FINALIZE) ||
304-
tp->tp_finalize == NULL)
301+
if (tp->tp_finalize == NULL)
305302
return;
306303
/* tp_finalize should only be called once. */
307304
if (PyType_IS_GC(tp) && _PyGC_FINALIZED(self))

‎Objects/typeobject.c

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -248,8 +248,8 @@ PyType_Modified(PyTypeObject *type)
248248
Invariants:
249249
250250
- Py_TPFLAGS_VALID_VERSION_TAG is never set if
251-
Py_TPFLAGS_HAVE_VERSION_TAG is not set (e.g. on type
252-
objects coming from non-recompiled extension modules)
251+
Py_TPFLAGS_HAVE_VERSION_TAG is not set (in case of a
252+
bizarre MRO, see type_mro_modified()).
253253
254254
- before Py_TPFLAGS_VALID_VERSION_TAG can be set on a type,
255255
it must first be set on all super types.
@@ -2571,7 +2571,7 @@ type_new(PyTypeObject *metatype, PyObject *args, PyObject *kwds)
25712571

25722572
/* Initialize tp_flags */
25732573
type->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HEAPTYPE |
2574-
Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_FINALIZE;
2574+
Py_TPFLAGS_BASETYPE;
25752575
if (base->tp_flags & Py_TPFLAGS_HAVE_GC)
25762576
type->tp_flags |= Py_TPFLAGS_HAVE_GC;
25772577

@@ -5179,10 +5179,7 @@ inherit_slots(PyTypeObject *type, PyTypeObject *base)
51795179
COPYSLOT(tp_init);
51805180
COPYSLOT(tp_alloc);
51815181
COPYSLOT(tp_is_gc);
5182-
if ((type->tp_flags & Py_TPFLAGS_HAVE_FINALIZE) &&
5183-
(base->tp_flags & Py_TPFLAGS_HAVE_FINALIZE)) {
5184-
COPYSLOT(tp_finalize);
5185-
}
5182+
COPYSLOT(tp_finalize);
51865183
if ((type->tp_flags & Py_TPFLAGS_HAVE_GC) ==
51875184
(base->tp_flags & Py_TPFLAGS_HAVE_GC)) {
51885185
/* They agree about gc. */

0 commit comments

Comments
 (0)