gh-84436: Implement Immortal Objects

Doc/library/sys.rst

@@ -670,6 +670,13 @@ always available.
    .. versionadded:: 3.4
 
 
+.. function:: getunicodeinternedsize()
+
+   Return the number of unicode objects that have been interned.
+
+   .. versionadded:: 3.12
+
+
 .. function:: getandroidapilevel()
 
    Return the build time API version of Android as an integer.

Doc/whatsnew/3.12.rst

@@ -1122,6 +1122,24 @@ New Features
   to replace the legacy-api :c:func:`!PyErr_Display`. (Contributed by
   Irit Katriel in :gh:`102755`).
 
+* :pep:`683`: Introduced Immortal Objects to Python which allows objects
+  to bypass reference counts and introduced changes to the C-API:
+
+  - ``_Py_IMMORTAL_REFCNT``: The reference count that defines an object
+      as immortal.
+  - ``_Py_IsImmortal`` Checks if an object has the immortal reference count.
+  - ``PyObject_HEAD_INIT`` This will now initialize reference count to
+      ``_Py_IMMORTAL_REFCNT`` when used with ``Py_BUILD_CORE``.
+  - ``SSTATE_INTERNED_IMMORTAL`` An identifier for interned unicode objects
+       that are immortal.
+  - ``SSTATE_INTERNED_IMMORTAL_STATIC`` An identifier for interned unicode
+       objects that are immortal and static
+  - ``sys.getunicodeinternedsize`` This returns the total number of unicode
+      objects that have been interned. This is now needed for refleak.py to
+      correctly track reference counts and allocated blocks
+
+  (Contributed by Eddie Elizondo in :gh:`84436`.)
+
 Porting to Python 3.12
 ----------------------
 
@@ -1286,8 +1304,7 @@ Removed
    * :c:func:`!PyUnicode_GetSize`
    * :c:func:`!PyUnicode_GET_DATA_SIZE`
 
-* Remove the ``PyUnicode_InternImmortal()`` function and the
-  ``SSTATE_INTERNED_IMMORTAL`` macro.
+* Remove the ``PyUnicode_InternImmortal()`` function macro.
   (Contributed by Victor Stinner in :gh:`85858`.)
 
 * Remove ``Jython`` compatibility hacks from several stdlib modules and tests.

Include/boolobject.h

@@ -11,8 +11,7 @@ PyAPI_DATA(PyTypeObject) PyBool_Type;
 
 #define PyBool_Check(x) Py_IS_TYPE((x), &PyBool_Type)
 
-/* Py_False and Py_True are the only two bools in existence.
-Don't forget to apply Py_INCREF() when returning either!!! */
+/* Py_False and Py_True are the only two bools in existence. */
 
 /* Don't use these directly */
 PyAPI_DATA(PyLongObject) _Py_FalseStruct;
@@ -31,8 +30,8 @@ PyAPI_FUNC(int) Py_IsFalse(PyObject *x);
 #define Py_IsFalse(x) Py_Is((x), Py_False)
 
 /* Macros for returning Py_True or Py_False, respectively */
-#define Py_RETURN_TRUE return Py_NewRef(Py_True)
-#define Py_RETURN_FALSE return Py_NewRef(Py_False)
+#define Py_RETURN_TRUE return Py_True
+#define Py_RETURN_FALSE return Py_False
 
 /* Function to return a bool from a C long */
 PyAPI_FUNC(PyObject *) PyBool_FromLong(long);

Include/cpython/unicodeobject.h

@@ -98,9 +98,16 @@ typedef struct {
     Py_ssize_t length;          /* Number of code points in the string */
     Py_hash_t hash;             /* Hash value; -1 if not set */
     struct {
-        /* If interned is set, the two references from the
-           dictionary to this object are *not* counted in ob_refcnt. */
-        unsigned int interned:1;
+        /* If interned is non-zero, the two references from the
+           dictionary to this object are *not* counted in ob_refcnt.
+           The possible values here are:
+               0: Not Interned
+               1: Interned
+               2: Interned and Immortal
+               3: Interned, Immortal, and Static
+           This categorization allows the runtime to determine the right
+           cleanup mechanism at runtime shutdown. */
+        unsigned int interned:2;
         /* Character size:
 
            - PyUnicode_1BYTE_KIND (1):
@@ -135,7 +142,7 @@ typedef struct {
         unsigned int ascii:1;
         /* Padding to ensure that PyUnicode_DATA() is always aligned to
            4 bytes (see issue #19537 on m68k). */
-        unsigned int :26;
+        unsigned int :25;
     } state;
 } PyASCIIObject;
 
@@ -183,6 +190,8 @@ PyAPI_FUNC(int) _PyUnicode_CheckConsistency(
 /* Interning state. */
 #define SSTATE_NOT_INTERNED 0
 #define SSTATE_INTERNED_MORTAL 1
+#define SSTATE_INTERNED_IMMORTAL 2
+#define SSTATE_INTERNED_IMMORTAL_STATIC 3
 
 /* Use only if you know it's a string */
 static inline unsigned int PyUnicode_CHECK_INTERNED(PyObject *op) {

Include/internal/pycore_long.h

@@ -245,7 +245,7 @@ _PyLong_FlipSign(PyLongObject *op) {
 
 #define _PyLong_DIGIT_INIT(val) \
     { \
-        .ob_base = _PyObject_IMMORTAL_INIT(&PyLong_Type), \
+        .ob_base = PyObject_HEAD_INIT(&PyLong_Type) \
         .long_value  = { \
             .lv_tag = TAG_FROM_SIGN_AND_SIZE( \
                 (val) == 0 ? 0 : ((val) < 0 ? -1 : 1), \

Include/internal/pycore_object.h

@@ -14,22 +14,6 @@ extern "C" {
 #include "pycore_pystate.h"       // _PyInterpreterState_GET()
 #include "pycore_runtime.h"       // _PyRuntime
 
-/* This value provides *effective* immortality, meaning the object should never
-    be deallocated (until runtime finalization).  See PEP 683 for more details about
-    immortality, as well as a proposed mechanism for proper immortality. */
-#define _PyObject_IMMORTAL_REFCNT 999999999
-
-#define _PyObject_IMMORTAL_INIT(type) \
-    { \
-        .ob_refcnt = _PyObject_IMMORTAL_REFCNT, \
-        .ob_type = (type), \
-    }
-#define _PyVarObject_IMMORTAL_INIT(type, size) \
-    { \
-        .ob_base = _PyObject_IMMORTAL_INIT(type), \
-        .ob_size = size, \
-    }
-
 PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalRefcountErrorFunc(
     const char *func,
     const char *message);
@@ -61,9 +45,20 @@ static inline void _Py_RefcntAdd(PyObject* op, Py_ssize_t n)
 }
 #define _Py_RefcntAdd(op, n) _Py_RefcntAdd(_PyObject_CAST(op), n)
 
+static inline void _Py_SetImmortal(PyObject *op)
+{
+    if (op) {
+        op->ob_refcnt = _Py_IMMORTAL_REFCNT;
+    }
+}
+#define _Py_SetImmortal(op) _Py_SetImmortal(_PyObject_CAST(op))
+
 static inline void
 _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct)
 {
+    if (_Py_IsImmortal(op)) {
+        return;
+    }
     _Py_DECREF_STAT_INC();
 #ifdef Py_REF_DEBUG
     _Py_DEC_REFTOTAL(_PyInterpreterState_GET());
@@ -82,6 +77,9 @@ _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct)
 static inline void
 _Py_DECREF_NO_DEALLOC(PyObject *op)
 {
+    if (_Py_IsImmortal(op)) {
+        return;
+    }
     _Py_DECREF_STAT_INC();
 #ifdef Py_REF_DEBUG
     _Py_DEC_REFTOTAL(_PyInterpreterState_GET());

Include/internal/pycore_runtime_init.h

@@ -76,13 +76,13 @@ extern PyTypeObject _PyExc_MemoryError;
                     .latin1 = _Py_str_latin1_INIT, \
                 }, \
                 .tuple_empty = { \
-                    .ob_base = _PyVarObject_IMMORTAL_INIT(&PyTuple_Type, 0) \
+                    .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, 0) \
                 }, \
                 .hamt_bitmap_node_empty = { \
-                    .ob_base = _PyVarObject_IMMORTAL_INIT(&_PyHamt_BitmapNode_Type, 0) \
+                    .ob_base = PyVarObject_HEAD_INIT(&_PyHamt_BitmapNode_Type, 0) \
                 }, \
                 .context_token_missing = { \
-                    .ob_base = _PyObject_IMMORTAL_INIT(&_PyContextTokenMissing_Type), \
+                    .ob_base = PyObject_HEAD_INIT(&_PyContextTokenMissing_Type) \
                 }, \
             }, \
         }, \
@@ -116,11 +116,11 @@ extern PyTypeObject _PyExc_MemoryError;
             .singletons = { \
                 ._not_used = 1, \
                 .hamt_empty = { \
-                    .ob_base = _PyObject_IMMORTAL_INIT(&_PyHamt_Type), \
+                    .ob_base = PyObject_HEAD_INIT(&_PyHamt_Type) \
                     .h_root = (PyHamtNode*)&_Py_SINGLETON(hamt_bitmap_node_empty), \
                 }, \
                 .last_resort_memory_error = { \
-                    _PyObject_IMMORTAL_INIT(&_PyExc_MemoryError), \
+                    PyObject_HEAD_INIT(&_PyExc_MemoryError) \
                 }, \
             }, \
         }, \
@@ -138,7 +138,7 @@ extern PyTypeObject _PyExc_MemoryError;
 
 #define _PyBytes_SIMPLE_INIT(CH, LEN) \
     { \
-        _PyVarObject_IMMORTAL_INIT(&PyBytes_Type, (LEN)), \
+        PyVarObject_HEAD_INIT(&PyBytes_Type, (LEN)) \
         .ob_shash = -1, \
         .ob_sval = { (CH) }, \
     }
@@ -149,7 +149,7 @@ extern PyTypeObject _PyExc_MemoryError;
 
 #define _PyUnicode_ASCII_BASE_INIT(LITERAL, ASCII) \
     { \
-        .ob_base = _PyObject_IMMORTAL_INIT(&PyUnicode_Type), \
+        .ob_base = PyObject_HEAD_INIT(&PyUnicode_Type) \
         .length = sizeof(LITERAL) - 1, \
         .hash = -1, \
         .state = { \

Include/internal/pycore_unicodeobject.h

@@ -12,6 +12,7 @@ extern "C" {
 #include "pycore_ucnhash.h"       // _PyUnicode_Name_CAPI
 
 void _PyUnicode_ExactDealloc(PyObject *op);
+Py_ssize_t _PyUnicode_InternedSize();
 
 /* runtime lifecycle */