Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

bpo-42927: Inline cache for slots #24216

Merged
merged 7 commits into from
Jan 30, 2021
Merged

bpo-42927: Inline cache for slots #24216

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
aca2939
Implement inline cache for slots
gvanrossum Jan 14, 2021
e334e53
Comment out debug print
gvanrossum Jan 15, 2021
ec0a590
Fall through to slow path if slot is NULL
gvanrossum Jan 15, 2021
b725a65
📜🤖 Added by blurb_it.
blurb-it[bot] Jan 15, 2021
4c2eb69
Fix the buildbot test failure (I hope).
gvanrossum Jan 28, 2021
17c8a95
Merge remote-tracking branch 'origin/master' into slotscache
gvanrossum Jan 28, 2021
86c4b39
Improve comment
gvanrossum Jan 28, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Misc/NEWS.d/next/Core and Builtins/2021-01-15-20-05-56.bpo-42927.GI-l-7.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
The inline cache for ``LOAD_ATTR`` now also optimizes access to attributes defined by ``__slots__``.
This makes reading such attribute up to 30% faster.
171 changes: 110 additions & 61 deletions Python/ceval.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
#include "opcode.h"
#include "pydtrace.h"
#include "setobject.h"
#include "structmember.h" // struct PyMemberDef, T_OFFSET_EX

#include <ctype.h>

Expand Down Expand Up @@ -3169,102 +3170,150 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag)
if (co_opcache != NULL && PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG))
{
if (co_opcache->optimized > 0) {
/* Fast path -- cache hit makes LOAD_ATTR ~30% faster */
// Fast path -- cache hit makes LOAD_ATTR ~30% faster.
la = &co_opcache->u.la;
if (la->type == type && la->tp_version_tag == type->tp_version_tag)
{
assert(type->tp_dict != NULL);
assert(type->tp_dictoffset > 0);

dictptr = (PyObject **) ((char *)owner + type->tp_dictoffset);
dict = *dictptr;
if (dict != NULL && PyDict_CheckExact(dict)) {
Py_ssize_t hint = la->hint;
Py_INCREF(dict);
res = NULL;
la->hint = _PyDict_GetItemHint((PyDictObject*)dict, name, hint, &res);

// Hint >= 0 is a dict index; hint < 0 is an inverted slot index.
if (la->hint < 0) {
/* Even faster path -- slot hint */
Py_ssize_t offset = ~la->hint;
// fprintf(stderr, "Using hint for offset %zd\n", offset);
char *addr = (char *)owner + offset;
res = *(PyObject **)addr;
if (res != NULL) {
if (la->hint == hint && hint >= 0) {
/* Our hint has helped -- cache hit. */
OPCACHE_STAT_ATTR_HIT();
} else {
/* The hint we provided didn't work.
Maybe next time? */
OPCACHE_MAYBE_DEOPT_LOAD_ATTR();
}

Py_INCREF(res);
SET_TOP(res);
Py_DECREF(owner);
Py_DECREF(dict);
DISPATCH();
}
// Else slot is NULL. Fall through to slow path to raise AttributeError(name).
// Don't DEOPT, since the slot is still there.
} else {
// Fast path for dict
assert(type->tp_dict != NULL);
assert(type->tp_dictoffset > 0);

dictptr = (PyObject **) ((char *)owner + type->tp_dictoffset);
dict = *dictptr;
if (dict != NULL && PyDict_CheckExact(dict)) {
Py_ssize_t hint = la->hint;
Py_INCREF(dict);
res = NULL;
la->hint = _PyDict_GetItemHint((PyDictObject*)dict, name, hint, &res);

if (res != NULL) {
if (la->hint == hint && hint >= 0) {
/* Our hint has helped -- cache hit. */
OPCACHE_STAT_ATTR_HIT();
} else {
/* The hint we provided didn't work.
Maybe next time? */
OPCACHE_MAYBE_DEOPT_LOAD_ATTR();
}

Py_INCREF(res);
SET_TOP(res);
Py_DECREF(owner);
Py_DECREF(dict);
DISPATCH();
} else {
// This attribute can be missing sometimes -- we
// don't want to optimize this lookup.
OPCACHE_DEOPT_LOAD_ATTR();
Py_DECREF(dict);
}
} else {
// This attribute can be missing sometimes -- we
// don't want to optimize this lookup.
// There is no dict, or __dict__ doesn't satisfy PyDict_CheckExact
OPCACHE_DEOPT_LOAD_ATTR();
Py_DECREF(dict);
}
} else {
// There is no dict, or __dict__ doesn't satisfy PyDict_CheckExact
OPCACHE_DEOPT_LOAD_ATTR();
}
} else {
// The type of the object has either been updated,
// or is different. Maybe it will stabilize?
OPCACHE_MAYBE_DEOPT_LOAD_ATTR();
}

OPCACHE_STAT_ATTR_MISS();
}

if (co_opcache != NULL && /* co_opcache can be NULL after a DEOPT() call. */
type->tp_getattro == PyObject_GenericGetAttr)
{
Py_ssize_t ret;

if (type->tp_dictoffset > 0) {
if (type->tp_dict == NULL) {
if (PyType_Ready(type) < 0) {
Py_DECREF(owner);
SET_TOP(NULL);
goto error;
}
if (type->tp_dict == NULL) {
if (PyType_Ready(type) < 0) {
Py_DECREF(owner);
SET_TOP(NULL);
goto error;
}
if (_PyType_Lookup(type, name) == NULL) {
dictptr = (PyObject **) ((char *)owner + type->tp_dictoffset);
dict = *dictptr;
}
PyObject *descr = _PyType_Lookup(type, name);
if (descr != NULL) {
// We found an attribute with a data-like descriptor.
PyTypeObject *dtype = Py_TYPE(descr);
if (dtype == &PyMemberDescr_Type) { // It's a slot
PyMemberDescrObject *member = (PyMemberDescrObject *)descr;
struct PyMemberDef *dmem = member->d_member;
if (dmem->type == T_OBJECT_EX) {
Py_ssize_t offset = dmem->offset;

if (co_opcache->optimized == 0) {
// First time we optimize this opcode.
OPCACHE_STAT_ATTR_OPT();
co_opcache->optimized = OPCODE_CACHE_MAX_TRIES;
// fprintf(stderr, "Setting hint for %s, offset %zd\n", dmem->name, offset);
}

if (dict != NULL && PyDict_CheckExact(dict)) {
Py_INCREF(dict);
res = NULL;
ret = _PyDict_GetItemHint((PyDictObject*)dict, name, -1, &res);
la = &co_opcache->u.la;
la->type = type;
la->tp_version_tag = type->tp_version_tag;
la->hint = ~offset;

char *addr = (char *)owner + offset;
res = *(PyObject **)addr;
if (res != NULL) {
Py_INCREF(res);
Py_DECREF(dict);
Py_DECREF(owner);
SET_TOP(res);

if (co_opcache->optimized == 0) {
// First time we optimize this opcode. */
OPCACHE_STAT_ATTR_OPT();
co_opcache->optimized = OPCODE_CACHE_MAX_TRIES;
}

la = &co_opcache->u.la;
la->type = type;
la->tp_version_tag = type->tp_version_tag;
la->hint = ret;

DISPATCH();
}
// Else slot is NULL. Fall through to slow path to raise AttributeError(name).
}
// Else it's a slot of a different type. We don't handle those.
}
// Else it's some other kind of descriptor that we don't handle.
OPCACHE_DEOPT_LOAD_ATTR();
} else if (type->tp_dictoffset > 0) {
// We found an instance with a __dict__.
dictptr = (PyObject **) ((char *)owner + type->tp_dictoffset);
dict = *dictptr;

if (dict != NULL && PyDict_CheckExact(dict)) {
Py_INCREF(dict);
res = NULL;
Py_ssize_t ret = _PyDict_GetItemHint((PyDictObject*)dict, name, -1, &res);
if (res != NULL) {
Py_INCREF(res);
Py_DECREF(dict);
} else {
// There is no dict, or __dict__ doesn't satisfy PyDict_CheckExact
OPCACHE_DEOPT_LOAD_ATTR();
Py_DECREF(owner);
SET_TOP(res);

if (co_opcache->optimized == 0) {
// First time we optimize this opcode.
OPCACHE_STAT_ATTR_OPT();
co_opcache->optimized = OPCODE_CACHE_MAX_TRIES;
}

la = &co_opcache->u.la;
la->type = type;
la->tp_version_tag = type->tp_version_tag;
la->hint = ret;

DISPATCH();
}
Py_DECREF(dict);
} else {
// We failed to find an attribute without a data-like descriptor
// There is no dict, or __dict__ doesn't satisfy PyDict_CheckExact
OPCACHE_DEOPT_LOAD_ATTR();
}
} else {
Expand Down