GH-100987: Allow objects other than code objects as the "executable" of an internal frame. (GH-105727)

* Add table describing possible executable classes for out-of-process debuggers.

* Remove shim code object creation code as it is no longer needed.

* Make lltrace a bit more robust w.r.t. non-standard frames.

Author: markshannon

Include/internal/pycore_code.h

@@ -447,19 +447,6 @@ adaptive_counter_backoff(uint16_t counter) {
     return adaptive_counter_bits(value, backoff);
 }
 
-
-/* Line array cache for tracing */
-
-typedef struct _PyShimCodeDef {
-    const uint8_t *code;
-    int codelen;
-    int stacksize;
-    const char *cname;
-} _PyShimCodeDef;
-
-extern PyCodeObject *
-_Py_MakeShimCode(const _PyShimCodeDef *code);
-
 extern uint32_t _Py_next_func_version;
 
 

Include/internal/pycore_frame.h

@@ -47,7 +47,7 @@ enum _frameowner {
 };
 
 typedef struct _PyInterpreterFrame {
-    PyCodeObject *f_code; /* Strong reference */
+    PyObject *f_executable; /* Strong reference */
     struct _PyInterpreterFrame *previous;
     PyObject *f_funcobj; /* Strong reference. Only valid if not on C stack */
     PyObject *f_globals; /* Borrowed reference. Only valid if not on C stack */
@@ -73,20 +73,25 @@ typedef struct _PyInterpreterFrame {
 } _PyInterpreterFrame;
 
 #define _PyInterpreterFrame_LASTI(IF) \
-    ((int)((IF)->prev_instr - _PyCode_CODE((IF)->f_code)))
+    ((int)((IF)->prev_instr - _PyCode_CODE(_PyFrame_GetCode(IF))))
+
+static inline PyCodeObject *_PyFrame_GetCode(_PyInterpreterFrame *f) {
+    assert(PyCode_Check(f->f_executable));
+    return (PyCodeObject *)f->f_executable;
+}
 
 static inline PyObject **_PyFrame_Stackbase(_PyInterpreterFrame *f) {
-    return f->localsplus + f->f_code->co_nlocalsplus;
+    return f->localsplus + _PyFrame_GetCode(f)->co_nlocalsplus;
 }
 
 static inline PyObject *_PyFrame_StackPeek(_PyInterpreterFrame *f) {
-    assert(f->stacktop > f->f_code->co_nlocalsplus);
+    assert(f->stacktop > _PyFrame_GetCode(f)->co_nlocalsplus);
     assert(f->localsplus[f->stacktop-1] != NULL);
     return f->localsplus[f->stacktop-1];
 }
 
 static inline PyObject *_PyFrame_StackPop(_PyInterpreterFrame *f) {
-    assert(f->stacktop > f->f_code->co_nlocalsplus);
+    assert(f->stacktop > _PyFrame_GetCode(f)->co_nlocalsplus);
     f->stacktop--;
     return f->localsplus[f->stacktop];
 }
@@ -119,7 +124,7 @@ _PyFrame_Initialize(
     PyObject *locals, PyCodeObject *code, int null_locals_from)
 {
     frame->f_funcobj = (PyObject *)func;
-    frame->f_code = (PyCodeObject *)Py_NewRef(code);
+    frame->f_executable = Py_NewRef(code);
     frame->f_builtins = func->func_builtins;
     frame->f_globals = func->func_globals;
     frame->f_locals = locals;
@@ -172,8 +177,11 @@ _PyFrame_SetStackPointer(_PyInterpreterFrame *frame, PyObject **stack_pointer)
 static inline bool
 _PyFrame_IsIncomplete(_PyInterpreterFrame *frame)
 {
+    if (frame->owner == FRAME_OWNED_BY_CSTACK) {
+        return true;
+    }
     return frame->owner != FRAME_OWNED_BY_GENERATOR &&
-    frame->prev_instr < _PyCode_CODE(frame->f_code) + frame->f_code->_co_firsttraceable;
+    frame->prev_instr < _PyCode_CODE(_PyFrame_GetCode(frame)) + _PyFrame_GetCode(frame)->_co_firsttraceable;
 }
 
 static inline _PyInterpreterFrame *
@@ -272,6 +280,14 @@ PyGenObject *_PyFrame_GetGenerator(_PyInterpreterFrame *frame)
     return (PyGenObject *)(((char *)frame) - offset_in_gen);
 }
 
+#define PY_EXECUTABLE_KIND_SKIP 0
+#define PY_EXECUTABLE_KIND_PY_FUNCTION 1
+#define PY_EXECUTABLE_KIND_BUILTIN_FUNCTION 3
+#define PY_EXECUTABLE_KIND_METHOD_DESCRIPTOR 4
+#define PY_EXECUTABLE_KINDS 5
+
+PyAPI_DATA(const PyTypeObject *) const PyUnstable_ExecutableKinds[PY_EXECUTABLE_KINDS+1];
+
 #ifdef __cplusplus
 }
 #endif

Include/internal/pycore_global_objects_fini_generated.h

@@ -51,7 +51,6 @@ struct _Py_global_strings {
         STRUCT_FOR_STR(newline, "\n")
         STRUCT_FOR_STR(open_br, "{")
         STRUCT_FOR_STR(percent, "%")
-        STRUCT_FOR_STR(shim_name, "<shim>")
         STRUCT_FOR_STR(type_params, ".type_params")
         STRUCT_FOR_STR(utf_8, "utf-8")
     } literals;

Include/internal/pycore_global_strings.h

@@ -159,7 +159,6 @@ struct _is {
     struct ast_state ast;
     struct types_state types;
     struct callable_cache callable_cache;
-    PyCodeObject *interpreter_trampoline;
     _PyOptimizerObject *optimizer;
     uint16_t optimizer_resume_threshold;
     uint16_t optimizer_backedge_threshold;

Include/internal/pycore_interp.h

@@ -0,0 +1,4 @@
+Allow objects other than code objects as the "executable" in internal
+frames. In the long term, this can help tools like Cython and PySpy interact
+more efficiently. In the shorter term, it allows us to perform some
+optimizations more simply.

Include/internal/pycore_runtime_init_generated.h

@@ -2316,76 +2316,3 @@ _PyStaticCode_Init(PyCodeObject *co)
 }
 
 #define MAX_CODE_UNITS_PER_LOC_ENTRY 8
-
-PyCodeObject *
-_Py_MakeShimCode(const _PyShimCodeDef *codedef)
-{
-    PyObject *name = NULL;
-    PyObject *co_code = NULL;
-    PyObject *lines = NULL;
-    PyCodeObject *codeobj = NULL;
-    uint8_t *loc_table = NULL;
-
-    name = _PyUnicode_FromASCII(codedef->cname, strlen(codedef->cname));
-    if (name == NULL) {
-        goto cleanup;
-    }
-    co_code = PyBytes_FromStringAndSize(
-        (const char *)codedef->code, codedef->codelen);
-    if (co_code == NULL) {
-        goto cleanup;
-    }
-    int code_units = codedef->codelen / sizeof(_Py_CODEUNIT);
-    int loc_entries = (code_units + MAX_CODE_UNITS_PER_LOC_ENTRY - 1) /
-                      MAX_CODE_UNITS_PER_LOC_ENTRY;
-    loc_table = PyMem_Malloc(loc_entries);
-    if (loc_table == NULL) {
-        PyErr_NoMemory();
-        goto cleanup;
-    }
-    for (int i = 0; i < loc_entries-1; i++) {
-         loc_table[i] = 0x80 | (PY_CODE_LOCATION_INFO_NONE << 3) | 7;
-         code_units -= MAX_CODE_UNITS_PER_LOC_ENTRY;
-    }
-    assert(loc_entries > 0);
-    assert(code_units > 0 && code_units <= MAX_CODE_UNITS_PER_LOC_ENTRY);
-    loc_table[loc_entries-1] = 0x80 |
-        (PY_CODE_LOCATION_INFO_NONE << 3) | (code_units-1);
-    lines = PyBytes_FromStringAndSize((const char *)loc_table, loc_entries);
-    PyMem_Free(loc_table);
-    if (lines == NULL) {
-        goto cleanup;
-    }
-    _Py_DECLARE_STR(shim_name, "<shim>");
-    struct _PyCodeConstructor con = {
-        .filename = &_Py_STR(shim_name),
-        .name = name,
-        .qualname = name,
-        .flags = CO_NEWLOCALS | CO_OPTIMIZED,
-
-        .code = co_code,
-        .firstlineno = 1,
-        .linetable = lines,
-
-        .consts = (PyObject *)&_Py_SINGLETON(tuple_empty),
-        .names = (PyObject *)&_Py_SINGLETON(tuple_empty),
-
-        .localsplusnames = (PyObject *)&_Py_SINGLETON(tuple_empty),
-        .localspluskinds = (PyObject *)&_Py_SINGLETON(bytes_empty),
-
-        .argcount = 0,
-        .posonlyargcount = 0,
-        .kwonlyargcount = 0,
-
-        .stacksize = codedef->stacksize,
-
-        .exceptiontable = (PyObject *)&_Py_SINGLETON(bytes_empty),
-    };
-
-    codeobj = _PyCode_New(&con);
-cleanup:
-    Py_XDECREF(name);
-    Py_XDECREF(co_code);
-    Py_XDECREF(lines);
-    return codeobj;
-}

Misc/NEWS.d/next/Core and Builtins/2023-06-09-10-48-17.gh-issue-100987.mK-xny.rst

@@ -642,6 +642,7 @@ _PyFrame_GetState(PyFrameObject *frame)
 static int
 frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignored))
 {
+    PyCodeObject *code = _PyFrame_GetCode(f->f_frame);
     if (p_new_lineno == NULL) {
         PyErr_SetString(PyExc_AttributeError, "cannot delete attribute");
         return -1;
@@ -719,7 +720,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore
     }
     new_lineno = (int)l_new_lineno;
 
-    if (new_lineno < f->f_frame->f_code->co_firstlineno) {
+    if (new_lineno < code->co_firstlineno) {
         PyErr_Format(PyExc_ValueError,
                     "line %d comes before the current code block",
                     new_lineno);
@@ -728,8 +729,8 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore
 
     /* PyCode_NewWithPosOnlyArgs limits co_code to be under INT_MAX so this
      * should never overflow. */
-    int len = (int)Py_SIZE(f->f_frame->f_code);
-    int *lines = marklines(f->f_frame->f_code, len);
+    int len = (int)Py_SIZE(code);
+    int *lines = marklines(code, len);
     if (lines == NULL) {
         return -1;
     }
@@ -743,7 +744,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore
         return -1;
     }
 
-    int64_t *stacks = mark_stacks(f->f_frame->f_code, len);
+    int64_t *stacks = mark_stacks(code, len);
     if (stacks == NULL) {
         PyMem_Free(lines);
         return -1;
@@ -788,7 +789,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore
     // in the new location. Rather than crashing or changing co_code, just bind
     // None instead:
     int unbound = 0;
-    for (int i = 0; i < f->f_frame->f_code->co_nlocalsplus; i++) {
+    for (int i = 0; i < code->co_nlocalsplus; i++) {
         // Counting every unbound local is overly-cautious, but a full flow
         // analysis (like we do in the compiler) is probably too expensive:
         unbound += f->f_frame->localsplus[i] == NULL;
@@ -801,7 +802,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore
         }
         // Do this in a second pass to avoid writing a bunch of Nones when
         // warnings are being treated as errors and the previous bit raises:
-        for (int i = 0; i < f->f_frame->f_code->co_nlocalsplus; i++) {
+        for (int i = 0; i < code->co_nlocalsplus; i++) {
             if (f->f_frame->localsplus[i] == NULL) {
                 f->f_frame->localsplus[i] = Py_NewRef(Py_None);
                 unbound--;
@@ -832,7 +833,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore
     }
     /* Finally set the new lasti and return OK. */
     f->f_lineno = 0;
-    f->f_frame->prev_instr = _PyCode_CODE(f->f_frame->f_code) + best_addr;
+    f->f_frame->prev_instr = _PyCode_CODE(code) + best_addr;
     return 0;
 }
 
@@ -886,15 +887,15 @@ frame_dealloc(PyFrameObject *f)
     }
 
     Py_TRASHCAN_BEGIN(f, frame_dealloc);
-    PyCodeObject *co = NULL;
+    PyObject *co = NULL;
 
     /* Kill all local variables including specials, if we own them */
     if (f->f_frame->owner == FRAME_OWNED_BY_FRAME_OBJECT) {
         assert(f->f_frame == (_PyInterpreterFrame *)f->_f_frame_data);
         _PyInterpreterFrame *frame = (_PyInterpreterFrame *)f->_f_frame_data;
         /* Don't clear code object until the end */
-        co = frame->f_code;
-        frame->f_code = NULL;
+        co = frame->f_executable;
+        frame->f_executable = NULL;
         Py_CLEAR(frame->f_funcobj);
         Py_CLEAR(frame->f_locals);
         PyObject **locals = _PyFrame_GetLocalsArray(frame);
@@ -968,7 +969,7 @@ frame_sizeof(PyFrameObject *f, PyObject *Py_UNUSED(ignored))
 {
     Py_ssize_t res;
     res = offsetof(PyFrameObject, _f_frame_data) + offsetof(_PyInterpreterFrame, localsplus);
-    PyCodeObject *code = f->f_frame->f_code;
+    PyCodeObject *code = _PyFrame_GetCode(f->f_frame);
     res += _PyFrame_NumSlotsForCodeObject(code) * sizeof(PyObject *);
     return PyLong_FromSsize_t(res);
 }
@@ -980,7 +981,7 @@ static PyObject *
 frame_repr(PyFrameObject *f)
 {
     int lineno = PyFrame_GetLineNumber(f);
-    PyCodeObject *code = f->f_frame->f_code;
+    PyCodeObject *code = _PyFrame_GetCode(f->f_frame);
     return PyUnicode_FromFormat(
         "<frame at %p, file %R, line %d, code %S>",
         f, code->co_filename, lineno, code->co_name);
@@ -1102,7 +1103,7 @@ _PyFrame_OpAlreadyRan(_PyInterpreterFrame *frame, int opcode, int oparg)
     // This only works when opcode is a non-quickened form:
     assert(_PyOpcode_Deopt[opcode] == opcode);
     int check_oparg = 0;
-    for (_Py_CODEUNIT *instruction = _PyCode_CODE(frame->f_code);
+    for (_Py_CODEUNIT *instruction = _PyCode_CODE(_PyFrame_GetCode(frame));
          instruction < frame->prev_instr; instruction++)
     {
         int check_opcode = _PyOpcode_Deopt[instruction->op.code];
@@ -1128,7 +1129,7 @@ frame_init_get_vars(_PyInterpreterFrame *frame)
 {
     // COPY_FREE_VARS has no quickened forms, so no need to use _PyOpcode_Deopt
     // here:
-    PyCodeObject *co = frame->f_code;
+    PyCodeObject *co = _PyFrame_GetCode(frame);
     int lasti = _PyInterpreterFrame_LASTI(frame);
     if (!(lasti < 0 && _PyCode_CODE(co)->op.code == COPY_FREE_VARS
           && PyFunction_Check(frame->f_funcobj)))
@@ -1145,7 +1146,7 @@ frame_init_get_vars(_PyInterpreterFrame *frame)
         frame->localsplus[offset + i] = Py_NewRef(o);
     }
     // COPY_FREE_VARS doesn't have inline CACHEs, either:
-    frame->prev_instr = _PyCode_CODE(frame->f_code);
+    frame->prev_instr = _PyCode_CODE(_PyFrame_GetCode(frame));
 }
 
 
@@ -1213,7 +1214,7 @@ _PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame)
 
     frame_init_get_vars(frame);
 
-    PyCodeObject *co = frame->f_code;
+    PyCodeObject *co = _PyFrame_GetCode(frame);
     for (int i = 0; i < co->co_nlocalsplus; i++) {
         PyObject *value;  // borrowed reference
         if (!frame_get_var(frame, co, i, &value)) {
@@ -1257,7 +1258,7 @@ PyFrame_GetVar(PyFrameObject *frame_obj, PyObject *name)
     _PyInterpreterFrame *frame = frame_obj->f_frame;
     frame_init_get_vars(frame);
 
-    PyCodeObject *co = frame->f_code;
+    PyCodeObject *co = _PyFrame_GetCode(frame);
     for (int i = 0; i < co->co_nlocalsplus; i++) {
         PyObject *var_name = PyTuple_GET_ITEM(co->co_localsplusnames, i);
         if (!_PyUnicode_Equal(var_name, name)) {
@@ -1331,7 +1332,7 @@ _PyFrame_LocalsToFast(_PyInterpreterFrame *frame, int clear)
         return;
     }
     fast = _PyFrame_GetLocalsArray(frame);
-    co = frame->f_code;
+    co = _PyFrame_GetCode(frame);
 
     PyObject *exc = PyErr_GetRaisedException();
     for (int i = 0; i < co->co_nlocalsplus; i++) {
@@ -1417,7 +1418,7 @@ PyFrame_GetCode(PyFrameObject *frame)
 {
     assert(frame != NULL);
     assert(!_PyFrame_IsIncomplete(frame->f_frame));
-    PyCodeObject *code = frame->f_frame->f_code;
+    PyCodeObject *code = _PyFrame_GetCode(frame->f_frame);
     assert(code != NULL);
     return (PyCodeObject*)Py_NewRef(code);
 }