bpo-42745: Make the type cache per-interpreter (GH-23947)
Make the type attribute lookup cache per-interpreter. Add private _PyType_InitCache() function, called by PyInterpreterState_New(). Continue to share next_version_tag between interpreters, since static types are still shared by interpreters. Remove MCACHE macro: the cache is no longer disabled if the EXPERIMENTAL_ISOLATED_SUBINTERPRETERS macro is defined.
This commit is contained in:
parent
77fde8dc16
commit
4101018488
|
@ -180,6 +180,27 @@ struct atexit_state {
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
|
// Type attribute lookup cache: speed up attribute and method lookups,
|
||||||
|
// see _PyType_Lookup().
|
||||||
|
struct type_cache_entry {
|
||||||
|
unsigned int version; // initialized from type->tp_version_tag
|
||||||
|
PyObject *name; // reference to exactly a str or None
|
||||||
|
PyObject *value; // borrowed reference or NULL
|
||||||
|
};
|
||||||
|
|
||||||
|
#define MCACHE_SIZE_EXP 12
|
||||||
|
#define MCACHE_STATS 0
|
||||||
|
|
||||||
|
struct type_cache {
|
||||||
|
struct type_cache_entry hashtable[1 << MCACHE_SIZE_EXP];
|
||||||
|
#if MCACHE_STATS
|
||||||
|
size_t hits;
|
||||||
|
size_t misses;
|
||||||
|
size_t collisions;
|
||||||
|
#endif
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
/* interpreter state */
|
/* interpreter state */
|
||||||
|
|
||||||
#define _PY_NSMALLPOSINTS 257
|
#define _PY_NSMALLPOSINTS 257
|
||||||
|
@ -284,6 +305,7 @@ struct _is {
|
||||||
struct _Py_exc_state exc_state;
|
struct _Py_exc_state exc_state;
|
||||||
|
|
||||||
struct ast_state ast;
|
struct ast_state ast;
|
||||||
|
struct type_cache type_cache;
|
||||||
};
|
};
|
||||||
|
|
||||||
extern void _PyInterpreterState_ClearModules(PyInterpreterState *interp);
|
extern void _PyInterpreterState_ClearModules(PyInterpreterState *interp);
|
||||||
|
|
|
@ -27,6 +27,9 @@ _PyType_HasFeature(PyTypeObject *type, unsigned long feature) {
|
||||||
return ((type->tp_flags & feature) != 0);
|
return ((type->tp_flags & feature) != 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
extern void _PyType_InitCache(PyInterpreterState *interp);
|
||||||
|
|
||||||
|
|
||||||
/* Inline functions trading binary compatibility for speed:
|
/* Inline functions trading binary compatibility for speed:
|
||||||
_PyObject_Init() is the fast version of PyObject_Init(), and
|
_PyObject_Init() is the fast version of PyObject_Init(), and
|
||||||
_PyObject_InitVar() is the fast version of PyObject_InitVar().
|
_PyObject_InitVar() is the fast version of PyObject_InitVar().
|
||||||
|
|
|
@ -76,7 +76,7 @@ extern void _PyExc_Fini(PyThreadState *tstate);
|
||||||
extern void _PyImport_Fini(void);
|
extern void _PyImport_Fini(void);
|
||||||
extern void _PyImport_Fini2(void);
|
extern void _PyImport_Fini2(void);
|
||||||
extern void _PyGC_Fini(PyThreadState *tstate);
|
extern void _PyGC_Fini(PyThreadState *tstate);
|
||||||
extern void _PyType_Fini(void);
|
extern void _PyType_Fini(PyThreadState *tstate);
|
||||||
extern void _Py_HashRandomization_Fini(void);
|
extern void _Py_HashRandomization_Fini(void);
|
||||||
extern void _PyUnicode_Fini(PyThreadState *tstate);
|
extern void _PyUnicode_Fini(PyThreadState *tstate);
|
||||||
extern void _PyUnicode_ClearInterned(PyThreadState *tstate);
|
extern void _PyUnicode_ClearInterned(PyThreadState *tstate);
|
||||||
|
|
|
@ -0,0 +1 @@
|
||||||
|
Make the type attribute lookup cache per-interpreter. Patch by Victor Stinner.
|
|
@ -20,20 +20,13 @@ class object "PyObject *" "&PyBaseObject_Type"
|
||||||
|
|
||||||
#include "clinic/typeobject.c.h"
|
#include "clinic/typeobject.c.h"
|
||||||
|
|
||||||
/* bpo-40521: Type method cache is shared by all subinterpreters */
|
/* Support type attribute lookup cache */
|
||||||
#ifndef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
|
|
||||||
# define MCACHE
|
|
||||||
#endif
|
|
||||||
|
|
||||||
#ifdef MCACHE
|
|
||||||
/* Support type attribute cache */
|
|
||||||
|
|
||||||
/* The cache can keep references to the names alive for longer than
|
/* The cache can keep references to the names alive for longer than
|
||||||
they normally would. This is why the maximum size is limited to
|
they normally would. This is why the maximum size is limited to
|
||||||
MCACHE_MAX_ATTR_SIZE, since it might be a problem if very large
|
MCACHE_MAX_ATTR_SIZE, since it might be a problem if very large
|
||||||
strings are used as attribute names. */
|
strings are used as attribute names. */
|
||||||
#define MCACHE_MAX_ATTR_SIZE 100
|
#define MCACHE_MAX_ATTR_SIZE 100
|
||||||
#define MCACHE_SIZE_EXP 12
|
|
||||||
#define MCACHE_HASH(version, name_hash) \
|
#define MCACHE_HASH(version, name_hash) \
|
||||||
(((unsigned int)(version) ^ (unsigned int)(name_hash)) \
|
(((unsigned int)(version) ^ (unsigned int)(name_hash)) \
|
||||||
& ((1 << MCACHE_SIZE_EXP) - 1))
|
& ((1 << MCACHE_SIZE_EXP) - 1))
|
||||||
|
@ -44,30 +37,16 @@ class object "PyObject *" "&PyBaseObject_Type"
|
||||||
#define MCACHE_CACHEABLE_NAME(name) \
|
#define MCACHE_CACHEABLE_NAME(name) \
|
||||||
PyUnicode_CheckExact(name) && \
|
PyUnicode_CheckExact(name) && \
|
||||||
PyUnicode_IS_READY(name) && \
|
PyUnicode_IS_READY(name) && \
|
||||||
PyUnicode_GET_LENGTH(name) <= MCACHE_MAX_ATTR_SIZE
|
(PyUnicode_GET_LENGTH(name) <= MCACHE_MAX_ATTR_SIZE)
|
||||||
|
|
||||||
struct method_cache_entry {
|
// Used to set PyTypeObject.tp_version_tag
|
||||||
unsigned int version;
|
|
||||||
PyObject *name; /* reference to exactly a str or None */
|
|
||||||
PyObject *value; /* borrowed */
|
|
||||||
};
|
|
||||||
|
|
||||||
static struct method_cache_entry method_cache[1 << MCACHE_SIZE_EXP];
|
|
||||||
static unsigned int next_version_tag = 0;
|
static unsigned int next_version_tag = 0;
|
||||||
#endif
|
|
||||||
|
|
||||||
typedef struct PySlot_Offset {
|
typedef struct PySlot_Offset {
|
||||||
short subslot_offset;
|
short subslot_offset;
|
||||||
short slot_offset;
|
short slot_offset;
|
||||||
} PySlot_Offset;
|
} PySlot_Offset;
|
||||||
|
|
||||||
#define MCACHE_STATS 0
|
|
||||||
|
|
||||||
#if MCACHE_STATS
|
|
||||||
static size_t method_cache_hits = 0;
|
|
||||||
static size_t method_cache_misses = 0;
|
|
||||||
static size_t method_cache_collisions = 0;
|
|
||||||
#endif
|
|
||||||
|
|
||||||
/* bpo-40521: Interned strings are shared by all subinterpreters */
|
/* bpo-40521: Interned strings are shared by all subinterpreters */
|
||||||
#ifndef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
|
#ifndef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS
|
||||||
|
@ -229,46 +208,93 @@ _PyType_GetTextSignatureFromInternalDoc(const char *name, const char *internal_d
|
||||||
return PyUnicode_FromStringAndSize(start, end - start);
|
return PyUnicode_FromStringAndSize(start, end - start);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
static struct type_cache*
|
||||||
|
get_type_cache(void)
|
||||||
|
{
|
||||||
|
PyInterpreterState *interp = _PyInterpreterState_GET();
|
||||||
|
return &interp->type_cache;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
static void
|
||||||
|
type_cache_clear(struct type_cache *cache, int use_none)
|
||||||
|
{
|
||||||
|
for (Py_ssize_t i = 0; i < (1 << MCACHE_SIZE_EXP); i++) {
|
||||||
|
struct type_cache_entry *entry = &cache->hashtable[i];
|
||||||
|
entry->version = 0;
|
||||||
|
if (use_none) {
|
||||||
|
// Set to None so _PyType_Lookup() can use Py_SETREF(),
|
||||||
|
// rather than using slower Py_XSETREF().
|
||||||
|
Py_XSETREF(entry->name, Py_NewRef(Py_None));
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
Py_CLEAR(entry->name);
|
||||||
|
}
|
||||||
|
entry->value = NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Mark all version tags as invalid
|
||||||
|
PyType_Modified(&PyBaseObject_Type);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
void
|
||||||
|
_PyType_InitCache(PyInterpreterState *interp)
|
||||||
|
{
|
||||||
|
struct type_cache *cache = &interp->type_cache;
|
||||||
|
for (Py_ssize_t i = 0; i < (1 << MCACHE_SIZE_EXP); i++) {
|
||||||
|
struct type_cache_entry *entry = &cache->hashtable[i];
|
||||||
|
assert(entry->name == NULL);
|
||||||
|
|
||||||
|
entry->version = 0;
|
||||||
|
// Set to None so _PyType_Lookup() can use Py_SETREF(),
|
||||||
|
// rather than using slower Py_XSETREF().
|
||||||
|
entry->name = Py_NewRef(Py_None);
|
||||||
|
entry->value = NULL;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
static unsigned int
|
||||||
|
_PyType_ClearCache(struct type_cache *cache)
|
||||||
|
{
|
||||||
|
#if MCACHE_STATS
|
||||||
|
size_t total = cache->hits + cache->collisions + cache->misses;
|
||||||
|
fprintf(stderr, "-- Method cache hits = %zd (%d%%)\n",
|
||||||
|
cache->hits, (int) (100.0 * cache->hits / total));
|
||||||
|
fprintf(stderr, "-- Method cache true misses = %zd (%d%%)\n",
|
||||||
|
cache->misses, (int) (100.0 * cache->misses / total));
|
||||||
|
fprintf(stderr, "-- Method cache collisions = %zd (%d%%)\n",
|
||||||
|
cache->collisions, (int) (100.0 * cache->collisions / total));
|
||||||
|
fprintf(stderr, "-- Method cache size = %zd KiB\n",
|
||||||
|
sizeof(cache->hashtable) / 1024);
|
||||||
|
#endif
|
||||||
|
|
||||||
|
unsigned int cur_version_tag = next_version_tag - 1;
|
||||||
|
next_version_tag = 0;
|
||||||
|
type_cache_clear(cache, 0);
|
||||||
|
|
||||||
|
return cur_version_tag;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
unsigned int
|
unsigned int
|
||||||
PyType_ClearCache(void)
|
PyType_ClearCache(void)
|
||||||
{
|
{
|
||||||
#ifdef MCACHE
|
struct type_cache *cache = get_type_cache();
|
||||||
Py_ssize_t i;
|
return _PyType_ClearCache(cache);
|
||||||
unsigned int cur_version_tag = next_version_tag - 1;
|
|
||||||
|
|
||||||
#if MCACHE_STATS
|
|
||||||
size_t total = method_cache_hits + method_cache_collisions + method_cache_misses;
|
|
||||||
fprintf(stderr, "-- Method cache hits = %zd (%d%%)\n",
|
|
||||||
method_cache_hits, (int) (100.0 * method_cache_hits / total));
|
|
||||||
fprintf(stderr, "-- Method cache true misses = %zd (%d%%)\n",
|
|
||||||
method_cache_misses, (int) (100.0 * method_cache_misses / total));
|
|
||||||
fprintf(stderr, "-- Method cache collisions = %zd (%d%%)\n",
|
|
||||||
method_cache_collisions, (int) (100.0 * method_cache_collisions / total));
|
|
||||||
fprintf(stderr, "-- Method cache size = %zd KiB\n",
|
|
||||||
sizeof(method_cache) / 1024);
|
|
||||||
#endif
|
|
||||||
|
|
||||||
for (i = 0; i < (1 << MCACHE_SIZE_EXP); i++) {
|
|
||||||
method_cache[i].version = 0;
|
|
||||||
Py_CLEAR(method_cache[i].name);
|
|
||||||
method_cache[i].value = NULL;
|
|
||||||
}
|
|
||||||
next_version_tag = 0;
|
|
||||||
/* mark all version tags as invalid */
|
|
||||||
PyType_Modified(&PyBaseObject_Type);
|
|
||||||
return cur_version_tag;
|
|
||||||
#else
|
|
||||||
return 0;
|
|
||||||
#endif
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void
|
void
|
||||||
_PyType_Fini(void)
|
_PyType_Fini(PyThreadState *tstate)
|
||||||
{
|
{
|
||||||
PyType_ClearCache();
|
_PyType_ClearCache(&tstate->interp->type_cache);
|
||||||
clear_slotdefs();
|
clear_slotdefs();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void
|
void
|
||||||
PyType_Modified(PyTypeObject *type)
|
PyType_Modified(PyTypeObject *type)
|
||||||
{
|
{
|
||||||
|
@ -370,9 +396,8 @@ type_mro_modified(PyTypeObject *type, PyObject *bases) {
|
||||||
Py_TPFLAGS_VALID_VERSION_TAG);
|
Py_TPFLAGS_VALID_VERSION_TAG);
|
||||||
}
|
}
|
||||||
|
|
||||||
#ifdef MCACHE
|
|
||||||
static int
|
static int
|
||||||
assign_version_tag(PyTypeObject *type)
|
assign_version_tag(struct type_cache *cache, PyTypeObject *type)
|
||||||
{
|
{
|
||||||
/* Ensure that the tp_version_tag is valid and set
|
/* Ensure that the tp_version_tag is valid and set
|
||||||
Py_TPFLAGS_VALID_VERSION_TAG. To respect the invariant, this
|
Py_TPFLAGS_VALID_VERSION_TAG. To respect the invariant, this
|
||||||
|
@ -393,31 +418,22 @@ assign_version_tag(PyTypeObject *type)
|
||||||
/* for stress-testing: next_version_tag &= 0xFF; */
|
/* for stress-testing: next_version_tag &= 0xFF; */
|
||||||
|
|
||||||
if (type->tp_version_tag == 0) {
|
if (type->tp_version_tag == 0) {
|
||||||
/* wrap-around or just starting Python - clear the whole
|
// Wrap-around or just starting Python - clear the whole cache
|
||||||
cache by filling names with references to Py_None.
|
type_cache_clear(cache, 1);
|
||||||
Values are also set to NULL for added protection, as they
|
|
||||||
are borrowed reference */
|
|
||||||
for (i = 0; i < (1 << MCACHE_SIZE_EXP); i++) {
|
|
||||||
method_cache[i].value = NULL;
|
|
||||||
Py_INCREF(Py_None);
|
|
||||||
Py_XSETREF(method_cache[i].name, Py_None);
|
|
||||||
}
|
|
||||||
/* mark all version tags as invalid */
|
|
||||||
PyType_Modified(&PyBaseObject_Type);
|
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
bases = type->tp_bases;
|
bases = type->tp_bases;
|
||||||
n = PyTuple_GET_SIZE(bases);
|
n = PyTuple_GET_SIZE(bases);
|
||||||
for (i = 0; i < n; i++) {
|
for (i = 0; i < n; i++) {
|
||||||
PyObject *b = PyTuple_GET_ITEM(bases, i);
|
PyObject *b = PyTuple_GET_ITEM(bases, i);
|
||||||
assert(PyType_Check(b));
|
assert(PyType_Check(b));
|
||||||
if (!assign_version_tag((PyTypeObject *)b))
|
if (!assign_version_tag(cache, (PyTypeObject *)b))
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
type->tp_flags |= Py_TPFLAGS_VALID_VERSION_TAG;
|
type->tp_flags |= Py_TPFLAGS_VALID_VERSION_TAG;
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
#endif
|
|
||||||
|
|
||||||
|
|
||||||
static PyMemberDef type_members[] = {
|
static PyMemberDef type_members[] = {
|
||||||
|
@ -3316,20 +3332,19 @@ _PyType_Lookup(PyTypeObject *type, PyObject *name)
|
||||||
PyObject *res;
|
PyObject *res;
|
||||||
int error;
|
int error;
|
||||||
|
|
||||||
#ifdef MCACHE
|
|
||||||
if (MCACHE_CACHEABLE_NAME(name) &&
|
if (MCACHE_CACHEABLE_NAME(name) &&
|
||||||
_PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG)) {
|
_PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG)) {
|
||||||
/* fast path */
|
/* fast path */
|
||||||
unsigned int h = MCACHE_HASH_METHOD(type, name);
|
unsigned int h = MCACHE_HASH_METHOD(type, name);
|
||||||
if (method_cache[h].version == type->tp_version_tag &&
|
struct type_cache *cache = get_type_cache();
|
||||||
method_cache[h].name == name) {
|
struct type_cache_entry *entry = &cache->hashtable[h];
|
||||||
|
if (entry->version == type->tp_version_tag && entry->name == name) {
|
||||||
#if MCACHE_STATS
|
#if MCACHE_STATS
|
||||||
method_cache_hits++;
|
cache->hits++;
|
||||||
#endif
|
#endif
|
||||||
return method_cache[h].value;
|
return entry->value;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
#endif
|
|
||||||
|
|
||||||
/* We may end up clearing live exceptions below, so make sure it's ours. */
|
/* We may end up clearing live exceptions below, so make sure it's ours. */
|
||||||
assert(!PyErr_Occurred());
|
assert(!PyErr_Occurred());
|
||||||
|
@ -3351,22 +3366,25 @@ _PyType_Lookup(PyTypeObject *type, PyObject *name)
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
#ifdef MCACHE
|
if (MCACHE_CACHEABLE_NAME(name)) {
|
||||||
if (MCACHE_CACHEABLE_NAME(name) && assign_version_tag(type)) {
|
struct type_cache *cache = get_type_cache();
|
||||||
unsigned int h = MCACHE_HASH_METHOD(type, name);
|
if (assign_version_tag(cache, type)) {
|
||||||
method_cache[h].version = type->tp_version_tag;
|
unsigned int h = MCACHE_HASH_METHOD(type, name);
|
||||||
method_cache[h].value = res; /* borrowed */
|
struct type_cache_entry *entry = &cache->hashtable[h];
|
||||||
Py_INCREF(name);
|
entry->version = type->tp_version_tag;
|
||||||
assert(((PyASCIIObject *)(name))->hash != -1);
|
entry->value = res; /* borrowed */
|
||||||
|
assert(((PyASCIIObject *)(name))->hash != -1);
|
||||||
#if MCACHE_STATS
|
#if MCACHE_STATS
|
||||||
if (method_cache[h].name != Py_None && method_cache[h].name != name)
|
if (entry->name != Py_None && entry->name != name) {
|
||||||
method_cache_collisions++;
|
cache->collisions++;
|
||||||
else
|
}
|
||||||
method_cache_misses++;
|
else {
|
||||||
|
cache->misses++;
|
||||||
|
}
|
||||||
#endif
|
#endif
|
||||||
Py_SETREF(method_cache[h].name, name);
|
Py_SETREF(entry->name, Py_NewRef(name));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
#endif
|
|
||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1750,7 +1750,7 @@ Py_FinalizeEx(void)
|
||||||
_PyImport_Fini();
|
_PyImport_Fini();
|
||||||
|
|
||||||
/* Cleanup typeobject.c's internal caches. */
|
/* Cleanup typeobject.c's internal caches. */
|
||||||
_PyType_Fini();
|
_PyType_Fini(tstate);
|
||||||
|
|
||||||
/* unload faulthandler module */
|
/* unload faulthandler module */
|
||||||
_PyFaulthandler_Fini();
|
_PyFaulthandler_Fini();
|
||||||
|
|
|
@ -4,6 +4,7 @@
|
||||||
#include "Python.h"
|
#include "Python.h"
|
||||||
#include "pycore_ceval.h"
|
#include "pycore_ceval.h"
|
||||||
#include "pycore_initconfig.h"
|
#include "pycore_initconfig.h"
|
||||||
|
#include "pycore_object.h" // _PyType_InitCache()
|
||||||
#include "pycore_pyerrors.h"
|
#include "pycore_pyerrors.h"
|
||||||
#include "pycore_pylifecycle.h"
|
#include "pycore_pylifecycle.h"
|
||||||
#include "pycore_pymem.h" // _PyMem_SetDefaultAllocator()
|
#include "pycore_pymem.h" // _PyMem_SetDefaultAllocator()
|
||||||
|
@ -223,6 +224,7 @@ PyInterpreterState_New(void)
|
||||||
|
|
||||||
_PyGC_InitState(&interp->gc);
|
_PyGC_InitState(&interp->gc);
|
||||||
PyConfig_InitPythonConfig(&interp->config);
|
PyConfig_InitPythonConfig(&interp->config);
|
||||||
|
_PyType_InitCache(interp);
|
||||||
|
|
||||||
interp->eval_frame = _PyEval_EvalFrameDefault;
|
interp->eval_frame = _PyEval_EvalFrameDefault;
|
||||||
#ifdef HAVE_DLOPEN
|
#ifdef HAVE_DLOPEN
|
||||||
|
|
Loading…
Reference in New Issue