Merge remote-tracking branch 'sunmy2019/memory-tracking-demo' into re…

…ftrace
JelleZijlstra · Apr 26, 2023 · 794f00b · 794f00b
2 parents 7fa37b4 + c6ca259
commit 794f00b
Show file tree

Hide file tree

Showing 14 changed files with 191 additions and 8 deletions.
diff --git a/Include/cpython/object.h b/Include/cpython/object.h
@@ -230,6 +230,10 @@ struct _typeobject {
     char tp_watched;
 };
 
+const char* __attribute__((weak)) get_type_name(PyTypeObject* type)  {
+    return type ? type->tp_name : "NULL";
+}
+
 /* This struct is used by the specializer
  * It should should be treated as an opaque blob
  * by code other than the specializer and interpreter. */

diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h
@@ -60,6 +60,7 @@ static inline void _Py_RefcntAdd(PyObject* op, Py_ssize_t n)
 {
 #ifdef Py_REF_DEBUG
     _Py_AddRefTotal(_PyInterpreterState_GET(), n);
+    state_change(op, op->ob_refcnt + n, +n, get_type_name(op->ob_type), _Py_GetGlobalRefTotal());
 #endif
     op->ob_refcnt += n;
 }
@@ -80,8 +81,9 @@ _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct)
         return;
     }
     _Py_DECREF_STAT_INC();
-#ifdef Py_REF_DEBUG
+#ifdef Py_REF_DEBUG 
     _Py_DEC_REFTOTAL(_PyInterpreterState_GET());
+    state_change(op, op->ob_refcnt - 1, -1, get_type_name(op->ob_type), _Py_GetGlobalRefTotal());
 #endif
     if (--op->ob_refcnt != 0) {
         assert(op->ob_refcnt > 0);
@@ -103,6 +105,7 @@ _Py_DECREF_NO_DEALLOC(PyObject *op)
     _Py_DECREF_STAT_INC();
 #ifdef Py_REF_DEBUG
     _Py_DEC_REFTOTAL(_PyInterpreterState_GET());
+    state_change(op, op->ob_refcnt - 1, -1, get_type_name(op->ob_type), _Py_GetGlobalRefTotal());
 #endif
     op->ob_refcnt--;
 #ifdef Py_DEBUG

diff --git a/Include/object.h b/Include/object.h
@@ -1,9 +1,16 @@
 #ifndef Py_OBJECT_H
 #define Py_OBJECT_H
+#include "pytypedefs.h"
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+extern long _Py_GetGlobalRefTotal(void);
+extern void state_change(void *ptr, long rc, long diff, const char *type,
+                         long current_total_ref);
+extern void store_state();
+extern void check_with_stored_state();
+extern const char* get_type_name(PyTypeObject* type);
 
 /* Object and type object interface */
 
@@ -247,6 +254,7 @@ static inline void Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) {
     if (_Py_IsImmortal(ob)) {
         return;
     }
+    state_change(ob, refcnt, 0, 0, _Py_GetGlobalRefTotal());
     ob->ob_refcnt = refcnt;
 }
 #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000
@@ -611,6 +619,9 @@ PyAPI_FUNC(void) _Py_DecRef(PyObject *);
 
 static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op)
 {
+#ifdef Py_REF_DEBUG
+    state_change(op, op->ob_refcnt + 1, +1, get_type_name(op->ob_type), _Py_GetGlobalRefTotal());
+#endif
 #if defined(Py_REF_DEBUG) && defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030A0000
     // Stable ABI for Python 3.10 built in debug mode.
     _Py_IncRef(op);
@@ -635,7 +646,7 @@ static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op)
     _Py_INCREF_STAT_INC();
 #ifdef Py_REF_DEBUG
     _Py_INC_REFTOTAL();
-#endif
+#endif  // Py_REF_DEBUG
 #endif
 }
 #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000
@@ -657,6 +668,7 @@ static inline void Py_DECREF(const char *filename, int lineno, PyObject *op)
     }
     _Py_DECREF_STAT_INC();
     _Py_DEC_REFTOTAL();
+    state_change(op, op->ob_refcnt - 1, -1, get_type_name(op->ob_type), _Py_GetGlobalRefTotal());
     if (--op->ob_refcnt != 0) {
         if (op->ob_refcnt < 0) {
             _Py_NegativeRefcount(filename, lineno, op);

diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c
@@ -24,6 +24,7 @@
 */
 
 #include "Python.h"
+#include "object.h"
 #include "pycore_context.h"
 #include "pycore_initconfig.h"
 #include "pycore_interp.h"      // PyInterpreterState.gc
@@ -2366,12 +2367,17 @@ _PyObject_GC_Resize(PyVarObject *op, Py_ssize_t nitems)
     if (basicsize > (size_t)PY_SSIZE_T_MAX - presize) {
         return (PyVarObject *)PyErr_NoMemory();
     }
+    long old_refcnt = op->ob_base.ob_refcnt;
+    state_change(op, 0, 0, op->ob_base.ob_type->tp_name,
+                 _Py_GetGlobalRefTotal());
     char *mem = (char *)op - presize;
     mem = (char *)PyObject_Realloc(mem,  presize + basicsize);
     if (mem == NULL) {
         return (PyVarObject *)PyErr_NoMemory();
     }
     op = (PyVarObject *) (mem + presize);
+    state_change(op, old_refcnt, 0, op->ob_base.ob_type->tp_name,
+                 _Py_GetGlobalRefTotal());
     Py_SET_SIZE(op, nitems);
     return op;
 }

diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c
@@ -3063,16 +3063,22 @@ _PyBytes_Resize(PyObject **pv, Py_ssize_t newsize)
 #ifdef Py_TRACE_REFS
     _Py_ForgetReference(v);
 #endif
+
+    long old_refcnt = v->ob_refcnt;
+    state_change(v, 0, 0, "bytes", _Py_GetGlobalRefTotal());
+
     *pv = (PyObject *)
         PyObject_Realloc(v, PyBytesObject_SIZE + newsize);
     if (*pv == NULL) {
 #ifdef Py_REF_DEBUG
-        _Py_DecRefTotal(_PyInterpreterState_GET());
+        _Py_DecRefTotal(_PyInterpreterState_GET()); 
 #endif
         PyObject_Free(v);
         PyErr_NoMemory();
         return -1;
     }
+    state_change(*pv, old_refcnt, 0, "bytes", _Py_GetGlobalRefTotal());
+
     _Py_NewReferenceNoTotal(*pv);
     sv = (PyBytesObject *) *pv;
     Py_SET_SIZE(sv, newsize);

diff --git a/Objects/dictobject.c b/Objects/dictobject.c
@@ -305,6 +305,7 @@ dictkeys_incref(PyDictKeysObject *dk)
 {
 #ifdef Py_REF_DEBUG
     _Py_IncRefTotal(_PyInterpreterState_GET());
+    state_change(dk, -1, +1, "__PyDictKeysObject", _Py_GetGlobalRefTotal());
 #endif
     dk->dk_refcnt++;
 }
@@ -315,6 +316,7 @@ dictkeys_decref(PyInterpreterState *interp, PyDictKeysObject *dk)
     assert(dk->dk_refcnt > 0);
 #ifdef Py_REF_DEBUG
     _Py_DecRefTotal(_PyInterpreterState_GET());
+    state_change(dk, -1, -1, "__PyDictKeysObject", _Py_GetGlobalRefTotal());
 #endif
     if (--dk->dk_refcnt == 0) {
         free_keys_object(interp, dk);
@@ -635,6 +637,7 @@ new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode)
     }
 #ifdef Py_REF_DEBUG
     _Py_IncRefTotal(_PyInterpreterState_GET());
+    state_change(dk, -1, +1, "__PyDictKeysObject", _Py_GetGlobalRefTotal());
 #endif
     dk->dk_refcnt = 1;
     dk->dk_log2_size = log2_size;
@@ -825,6 +828,7 @@ clone_combined_dict_keys(PyDictObject *orig)
        keys->dk_refcnt is already set to 1 (after memcpy). */
 #ifdef Py_REF_DEBUG
     _Py_IncRefTotal(_PyInterpreterState_GET());
+    state_change(keys, -1, +1, "__PyDictKeysObject", _Py_GetGlobalRefTotal());
 #endif
     return keys;
 }
@@ -1531,6 +1535,7 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp,
         // are moved already.
 #ifdef Py_REF_DEBUG
         _Py_DecRefTotal(_PyInterpreterState_GET());
+        state_change(oldkeys, -1, -1, "__PyDictKeysObject", _Py_GetGlobalRefTotal());
 #endif
         if (oldkeys == Py_EMPTY_KEYS) {
             oldkeys->dk_refcnt--;

diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c
@@ -42,8 +42,8 @@ PyModuleDef_Init(PyModuleDef* def)
 {
     assert(PyModuleDef_Type.tp_flags & Py_TPFLAGS_READY);
     if (def->m_base.m_index == 0) {
-        Py_SET_REFCNT(def, 1);
         Py_SET_TYPE(def, &PyModuleDef_Type);
+        Py_SET_REFCNT(def, 1);
         def->m_base.m_index = _PyImport_GetNextModuleIndex();
     }
     return (PyObject*)def;

diff --git a/Objects/object.c b/Objects/object.c
@@ -73,6 +73,9 @@ get_legacy_reftotal(void)
 static inline void
 reftotal_increment(PyInterpreterState *interp)
 {
+    // if ( REFTOTAL(interp) >= 26600){
+    //     fprintf(stderr, "here -----> %p\n", &(  REFTOTAL(interp) ));
+    // }
     REFTOTAL(interp)++;
 }
 
@@ -2166,6 +2169,7 @@ _Py_NewReference(PyObject *op)
 {
 #ifdef Py_REF_DEBUG
     reftotal_increment(_PyInterpreterState_GET());
+    state_change(op, 1, 1, op->ob_type->tp_name, _Py_GetGlobalRefTotal());
 #endif
     new_reference(op);
 }

diff --git a/Objects/structseq.c b/Objects/structseq.c
@@ -592,7 +592,8 @@ _PyStructSequence_FiniType(PyTypeObject *type)
     // Don't use Py_DECREF(): static type must not be deallocated
     Py_SET_REFCNT(type, 0);
 #ifdef Py_REF_DEBUG
-    _Py_DecRefTotal(_PyInterpreterState_GET());
+    _Py_DecRefTotal(_PyInterpreterState_GET());   
+    state_change(type, 0, -1, "type", _Py_GetGlobalRefTotal());
 #endif
 
     // Make sure that _PyStructSequence_InitType() will initialize

diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c
@@ -945,6 +945,7 @@ _PyTuple_Resize(PyObject **pv, Py_ssize_t newsize)
         *pv = NULL;
 #ifdef Py_REF_DEBUG
         _Py_DecRefTotal(_PyInterpreterState_GET());
+        state_change(v, 0, -1, "tuple", _Py_GetGlobalRefTotal());
 #endif
         PyObject_GC_Del(v);
         return -1;

diff --git a/Objects/typeobject.c b/Objects/typeobject.c
@@ -2171,7 +2171,6 @@ mro_implementation(PyTypeObject *type)
             return NULL;
         }
 
-        ;
         PyTuple_SET_ITEM(result, 0, Py_NewRef(type));
         for (Py_ssize_t i = 0; i < k; i++) {
             PyObject *cls = PyTuple_GET_ITEM(base->tp_mro, i);

diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
@@ -56,6 +56,7 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 #include "pycore_unicodeobject.h" // struct _Py_unicode_state
 #include "pycore_unicodeobject_generated.h"  // _PyUnicode_InitStaticStrings()
 #include "stringlib/eq.h"         // unicode_eq()
+#include "object.h"
 
 #ifdef MS_WINDOWS
 #include <windows.h>
@@ -972,14 +973,17 @@ resize_compact(PyObject *unicode, Py_ssize_t length)
 #ifdef Py_TRACE_REFS
     _Py_ForgetReference(unicode);
 #endif
-
+    long old_refcnt = unicode->ob_refcnt;
+    state_change(unicode, 0, 0, "str", _Py_GetGlobalRefTotal());
     new_unicode = (PyObject *)PyObject_Realloc(unicode, new_size);
     if (new_unicode == NULL) {
         _Py_NewReferenceNoTotal(unicode);
         PyErr_NoMemory();
         return NULL;
     }
     unicode = new_unicode;
+    state_change(unicode, old_refcnt, 0, "str", _Py_GetGlobalRefTotal());
+
     _Py_NewReferenceNoTotal(unicode);
 
     _PyUnicode_LENGTH(unicode) = length;
@@ -1022,12 +1026,15 @@ resize_inplace(PyObject *unicode, Py_ssize_t length)
         _PyUnicode_UTF8(unicode) = NULL;
         _PyUnicode_UTF8_LENGTH(unicode) = 0;
     }
-
+    long old_refcnt = unicode->ob_refcnt;
+    state_change(unicode, 0, 0, "str", _Py_GetGlobalRefTotal());
     data = (PyObject *)PyObject_Realloc(data, new_size);
     if (data == NULL) {
         PyErr_NoMemory();
         return -1;
     }
+    state_change(unicode, old_refcnt, 0, "str", _Py_GetGlobalRefTotal());
+
     _PyUnicode_DATA_ANY(unicode) = data;
     if (share_utf8) {
         _PyUnicode_UTF8(unicode) = data;

diff --git a/Python/sysmodule.c b/Python/sysmodule.c
@@ -36,6 +36,7 @@ Data members:
 #include "osdefs.h"               // DELIM
 #include "stdlib_module_names.h"  // _Py_stdlib_module_names
 #include <locale.h>
+#include <stdio.h>
 
 #ifdef MS_WINDOWS
 #define WIN32_LEAN_AND_MEAN
@@ -1850,12 +1851,18 @@ sys_getrefcount_impl(PyObject *module, PyObject *object)
 sys.gettotalrefcount -> Py_ssize_t
 [clinic start generated code]*/
 
+extern void store_state();
+extern void check_with_stored_state();
+
 static Py_ssize_t
 sys_gettotalrefcount_impl(PyObject *module)
 /*[clinic end generated code: output=4103886cf17c25bc input=53b744faa5d2e4f6]*/
 {
     /* It may make sense to return the total for the current interpreter
        or have a second function that does so. */
+    check_with_stored_state();
+    store_state();
+    fprintf(stderr, "ref total %ld\n", _Py_GetGlobalRefTotal());
     return _Py_GetGlobalRefTotal();
 }