GH-115776: Embed the values array into the object, for "normal" Python objects. (GH-116115)

Include/cpython/pystats.h

@@ -77,12 +77,11 @@ typedef struct _object_stats {
     uint64_t frees;
     uint64_t to_freelist;
     uint64_t from_freelist;
-    uint64_t new_values;
+    uint64_t inline_values;
     uint64_t dict_materialized_on_request;
     uint64_t dict_materialized_new_key;
     uint64_t dict_materialized_too_big;
     uint64_t dict_materialized_str_subclass;
-    uint64_t dict_dematerialized;
     uint64_t type_cache_hits;
     uint64_t type_cache_misses;
     uint64_t type_cache_dunder_hits;

Include/internal/pycore_code.h

@@ -79,7 +79,10 @@ typedef struct {
 typedef struct {
     uint16_t counter;
     uint16_t type_version[2];
+    union {
         uint16_t keys_version[2];
+        uint16_t dict_offset;
+    };
     uint16_t descr[4];
 } _PyLoadMethodCache;
 

Include/internal/pycore_dict.h

@@ -11,7 +11,7 @@ extern "C" {
 
 #include "pycore_freelist.h"      // _PyFreeListState
 #include "pycore_identifier.h"    // _Py_Identifier
-#include "pycore_object.h"        // PyDictOrValues
+#include "pycore_object.h"        // PyManagedDictPointer
 
 // Unsafe flavor of PyDict_GetItemWithError(): no error checking
 extern PyObject* _PyDict_GetItemWithError(PyObject *dp, PyObject *key);
@@ -181,6 +181,10 @@ struct _dictkeysobject {
  * [-1] = prefix size. [-2] = used size. size[-2-n...] = insertion order.
  */
 struct _dictvalues {
+    uint8_t capacity;
+    uint8_t size;
+    uint8_t embedded;
+    uint8_t valid;
     PyObject *values[1];
 };
 
@@ -196,6 +200,7 @@ static inline void* _DK_ENTRIES(PyDictKeysObject *dk) {
     size_t index = (size_t)1 << dk->dk_log2_index_bytes;
     return (&indices[index]);
 }
+
 static inline PyDictKeyEntry* DK_ENTRIES(PyDictKeysObject *dk) {
     assert(dk->dk_kind == DICT_KEYS_GENERAL);
     return (PyDictKeyEntry*)_DK_ENTRIES(dk);
@@ -211,9 +216,6 @@ static inline PyDictUnicodeEntry* DK_UNICODE_ENTRIES(PyDictKeysObject *dk) {
 #define DICT_WATCHER_MASK ((1 << DICT_MAX_WATCHERS) - 1)
 #define DICT_WATCHER_AND_MODIFICATION_MASK ((1 << (DICT_MAX_WATCHERS + DICT_WATCHED_MUTATION_BITS)) - 1)
 
-#define DICT_VALUES_SIZE(values) ((uint8_t *)values)[-1]
-#define DICT_VALUES_USED_SIZE(values) ((uint8_t *)values)[-2]
-
 #ifdef Py_GIL_DISABLED
 #define DICT_NEXT_VERSION(INTERP) \
     (_Py_atomic_add_uint64(&(INTERP)->dict_state.global_version, DICT_VERSION_INCREMENT) + DICT_VERSION_INCREMENT)
@@ -246,25 +248,63 @@ _PyDict_NotifyEvent(PyInterpreterState *interp,
     return DICT_NEXT_VERSION(interp) | (mp->ma_version_tag & DICT_WATCHER_AND_MODIFICATION_MASK);
 }
 
-extern PyObject *_PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values);
+extern PyDictObject *_PyObject_MakeDictFromInstanceAttributes(PyObject *obj);
-PyAPI_FUNC(bool) _PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv);
+
 PyAPI_FUNC(PyObject *)_PyDict_FromItems(
         PyObject *const *keys, Py_ssize_t keys_offset,
         PyObject *const *values, Py_ssize_t values_offset,
         Py_ssize_t length);
 
+static inline uint8_t *
+get_insertion_order_array(PyDictValues *values)
+{
+    return (uint8_t *)&values->values[values->capacity];
+}
+
 static inline void
 _PyDictValues_AddToInsertionOrder(PyDictValues *values, Py_ssize_t ix)
 {
     assert(ix < SHARED_KEYS_MAX_SIZE);
-    uint8_t *size_ptr = ((uint8_t *)values)-2;
+    int size = values->size;
-    int size = *size_ptr;
+    uint8_t *array = get_insertion_order_array(values);
-    assert(size+2 < DICT_VALUES_SIZE(values));
+    assert(size < values->capacity);
-    size++;
+    assert(((uint8_t)ix) == ix);
-    size_ptr[-size] = (uint8_t)ix;
+    array[size] = (uint8_t)ix;
-    *size_ptr = size;
+    values->size = size+1;
 }
 
+static inline size_t
+shared_keys_usable_size(PyDictKeysObject *keys)
+{
+#ifdef Py_GIL_DISABLED
+    // dk_usable will decrease for each instance that is created and each
+    // value that is added.  dk_nentries will increase for each value that
+    // is added.  We want to always return the right value or larger.
+    // We therefore increase dk_nentries first and we decrease dk_usable
+    // second, and conversely here we read dk_usable first and dk_entries
+    // second (to avoid the case where we read entries before the increment
+    // and read usable after the decrement)
+    return (size_t)(_Py_atomic_load_ssize_acquire(&keys->dk_usable) +
+                    _Py_atomic_load_ssize_acquire(&keys->dk_nentries));
+#else
+    return (size_t)keys->dk_nentries + (size_t)keys->dk_usable;
+#endif
+}
+
+static inline size_t
+_PyInlineValuesSize(PyTypeObject *tp)
+{
+    PyDictKeysObject *keys = ((PyHeapTypeObject*)tp)->ht_cached_keys;
+    assert(keys != NULL);
+    size_t size = shared_keys_usable_size(keys);
+    size_t prefix_size = _Py_SIZE_ROUND_UP(size, sizeof(PyObject *));
+    assert(prefix_size < 256);
+    return prefix_size + (size + 1) * sizeof(PyObject *);
+}
+
+int
+_PyDict_DetachFromObject(PyDictObject *dict, PyObject *obj);
+
 #ifdef __cplusplus
 }
 #endif

Include/internal/pycore_object.h

@@ -265,9 +265,8 @@ _PyObject_Init(PyObject *op, PyTypeObject *typeobj)
 {
     assert(op != NULL);
     Py_SET_TYPE(op, typeobj);
-    if (_PyType_HasFeature(typeobj, Py_TPFLAGS_HEAPTYPE)) {
+    assert(_PyType_HasFeature(typeobj, Py_TPFLAGS_HEAPTYPE) || _Py_IsImmortal(typeobj));
     Py_INCREF(typeobj);
-    }
     _Py_NewReference(op);
 }
 
@@ -611,8 +610,7 @@ extern PyTypeObject* _PyType_CalculateMetaclass(PyTypeObject *, PyObject *);
 extern PyObject* _PyType_GetDocFromInternalDoc(const char *, const char *);
 extern PyObject* _PyType_GetTextSignatureFromInternalDoc(const char *, const char *, int);
 
-extern int _PyObject_InitializeDict(PyObject *obj);
+void _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp);
-int _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp);
 extern int _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values,
                                           PyObject *name, PyObject *value);
 PyObject * _PyObject_GetInstanceAttribute(PyObject *obj, PyDictValues *values,
@@ -627,46 +625,26 @@ PyObject * _PyObject_GetInstanceAttribute(PyObject *obj, PyDictValues *values,
 #endif
 
 typedef union {
-    PyObject *dict;
+    PyDictObject *dict;
-    /* Use a char* to generate a warning if directly assigning a PyDictValues */
+} PyManagedDictPointer;
-    char *values;
-} PyDictOrValues;
 
-static inline PyDictOrValues *
+static inline PyManagedDictPointer *
-_PyObject_DictOrValuesPointer(PyObject *obj)
+_PyObject_ManagedDictPointer(PyObject *obj)
 {
     assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
-    return (PyDictOrValues *)((char *)obj + MANAGED_DICT_OFFSET);
+    return (PyManagedDictPointer *)((char *)obj + MANAGED_DICT_OFFSET);
-}
-
-static inline int
-_PyDictOrValues_IsValues(PyDictOrValues dorv)
-{
-    return ((uintptr_t)dorv.values) & 1;
 }
 
 static inline PyDictValues *
-_PyDictOrValues_GetValues(PyDictOrValues dorv)
+_PyObject_InlineValues(PyObject *obj)
 {
-    assert(_PyDictOrValues_IsValues(dorv));
+    assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
-    return (PyDictValues *)(dorv.values + 1);
+    assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
-}
+    assert(Py_TYPE(obj)->tp_basicsize == sizeof(PyObject));
-
+    return (PyDictValues *)((char *)obj + sizeof(PyObject));
-static inline PyObject *
-_PyDictOrValues_GetDict(PyDictOrValues dorv)
-{
-    assert(!_PyDictOrValues_IsValues(dorv));
-    return dorv.dict;
-}
-
-static inline void
-_PyDictOrValues_SetValues(PyDictOrValues *ptr, PyDictValues *values)
-{
-    ptr->values = ((char *)values) - 1;
 }
 
 extern PyObject ** _PyObject_ComputedDictPointer(PyObject *);
-extern void _PyObject_FreeInstanceAttributes(PyObject *obj);
 extern int _PyObject_IsInstanceDictEmpty(PyObject *);
 
 // Export for 'math' shared extension

Include/internal/pycore_opcode_metadata.h

@@ -1085,7 +1085,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = {
     [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
     [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG },
     [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
-    [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG },
+    [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
     [LOAD_BUILD_CLASS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
     [LOAD_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_PURE_FLAG },
     [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
@@ -1269,7 +1269,7 @@ _PyOpcode_macro_expansion[256] = {
     [LOAD_ATTR] = { .nuops = 1, .uops = { { _LOAD_ATTR, 0, 0 } } },
     [LOAD_ATTR_CLASS] = { .nuops = 2, .uops = { { _CHECK_ATTR_CLASS, 2, 1 }, { _LOAD_ATTR_CLASS, 4, 5 } } },
     [LOAD_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, 0, 0 }, { _LOAD_ATTR_INSTANCE_VALUE, 1, 3 } } },
-    [LOAD_ATTR_METHOD_LAZY_DICT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_METHOD_LAZY_DICT, 0, 0 }, { _LOAD_ATTR_METHOD_LAZY_DICT, 4, 5 } } },
+    [LOAD_ATTR_METHOD_LAZY_DICT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_METHOD_LAZY_DICT, 1, 3 }, { _LOAD_ATTR_METHOD_LAZY_DICT, 4, 5 } } },
     [LOAD_ATTR_METHOD_NO_DICT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_METHOD_NO_DICT, 4, 5 } } },
     [LOAD_ATTR_METHOD_WITH_VALUES] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, 0, 0 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_METHOD_WITH_VALUES, 4, 5 } } },
     [LOAD_ATTR_MODULE] = { .nuops = 2, .uops = { { _CHECK_ATTR_MODULE, 2, 1 }, { _LOAD_ATTR_MODULE, 1, 3 } } },
@@ -1316,7 +1316,7 @@ _PyOpcode_macro_expansion[256] = {
     [SET_FUNCTION_ATTRIBUTE] = { .nuops = 1, .uops = { { _SET_FUNCTION_ATTRIBUTE, 0, 0 } } },
     [SET_UPDATE] = { .nuops = 1, .uops = { { _SET_UPDATE, 0, 0 } } },
     [STORE_ATTR] = { .nuops = 1, .uops = { { _STORE_ATTR, 0, 0 } } },
-    [STORE_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES, 0, 0 }, { _STORE_ATTR_INSTANCE_VALUE, 1, 3 } } },
+    [STORE_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_NO_DICT, 0, 0 }, { _STORE_ATTR_INSTANCE_VALUE, 1, 3 } } },
     [STORE_ATTR_SLOT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_SLOT, 1, 3 } } },
     [STORE_DEREF] = { .nuops = 1, .uops = { { _STORE_DEREF, 0, 0 } } },
     [STORE_FAST] = { .nuops = 1, .uops = { { _STORE_FAST, 0, 0 } } },

Include/internal/pycore_uop_ids.h

@@ -109,7 +109,7 @@ extern "C" {
 #define _GUARD_BOTH_INT 347
 #define _GUARD_BOTH_UNICODE 348
 #define _GUARD_BUILTINS_VERSION 349
-#define _GUARD_DORV_VALUES 350
+#define _GUARD_DORV_NO_DICT 350
 #define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 351
 #define _GUARD_GLOBALS_VERSION 352
 #define _GUARD_IS_FALSE_POP 353

Include/internal/pycore_uop_metadata.h

@@ -136,8 +136,8 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {
     [_LOAD_ATTR_INSTANCE_VALUE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_OPARG_AND_1_FLAG,
     [_CHECK_ATTR_MODULE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG,
     [_LOAD_ATTR_MODULE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG,
-    [_CHECK_ATTR_WITH_HINT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG | HAS_PASSTHROUGH_FLAG,
+    [_CHECK_ATTR_WITH_HINT] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG,
-    [_LOAD_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG,
+    [_LOAD_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG,
     [_LOAD_ATTR_SLOT_0] = HAS_DEOPT_FLAG,
     [_LOAD_ATTR_SLOT_1] = HAS_DEOPT_FLAG,
     [_LOAD_ATTR_SLOT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_OPARG_AND_1_FLAG,
@@ -145,7 +145,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {
     [_LOAD_ATTR_CLASS_0] = 0,
     [_LOAD_ATTR_CLASS_1] = 0,
     [_LOAD_ATTR_CLASS] = HAS_ARG_FLAG | HAS_OPARG_AND_1_FLAG,
-    [_GUARD_DORV_VALUES] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG,
+    [_GUARD_DORV_NO_DICT] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG,
     [_STORE_ATTR_INSTANCE_VALUE] = 0,
     [_STORE_ATTR_SLOT] = HAS_ESCAPES_FLAG,
     [_COMPARE_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
@@ -342,7 +342,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = {
     [_GUARD_BOTH_INT] = "_GUARD_BOTH_INT",
     [_GUARD_BOTH_UNICODE] = "_GUARD_BOTH_UNICODE",
     [_GUARD_BUILTINS_VERSION] = "_GUARD_BUILTINS_VERSION",
-    [_GUARD_DORV_VALUES] = "_GUARD_DORV_VALUES",
+    [_GUARD_DORV_NO_DICT] = "_GUARD_DORV_NO_DICT",
     [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT",
     [_GUARD_GLOBALS_VERSION] = "_GUARD_GLOBALS_VERSION",
     [_GUARD_IS_FALSE_POP] = "_GUARD_IS_FALSE_POP",
@@ -736,7 +736,7 @@ int _PyUop_num_popped(int opcode, int oparg)
             return 1;
         case _LOAD_ATTR_CLASS:
             return 1;
-        case _GUARD_DORV_VALUES:
+        case _GUARD_DORV_NO_DICT:
             return 1;
         case _STORE_ATTR_INSTANCE_VALUE:
             return 2;

Include/object.h

@@ -629,13 +629,18 @@ given type object has a specified feature.
 /* Track types initialized using _PyStaticType_InitBuiltin(). */
 #define _Py_TPFLAGS_STATIC_BUILTIN (1 << 1)
 
+/* The values array is placed inline directly after the rest of
+ * the object. Implies Py_TPFLAGS_HAVE_GC.
+ */
+#define Py_TPFLAGS_INLINE_VALUES (1 << 2)
+
 /* Placement of weakref pointers are managed by the VM, not by the type.
  * The VM will automatically set tp_weaklistoffset.
  */
 #define Py_TPFLAGS_MANAGED_WEAKREF (1 << 3)
 
 /* Placement of dict (and values) pointers are managed by the VM, not by the type.
- * The VM will automatically set tp_dictoffset.
+ * The VM will automatically set tp_dictoffset. Implies Py_TPFLAGS_HAVE_GC.
  */
 #define Py_TPFLAGS_MANAGED_DICT (1 << 4)
 

Lib/test/test_capi/test_mem.py

@@ -148,8 +148,8 @@ class PyMemDebugTests(unittest.TestCase):
             self.assertIn(b'MemoryError', out)
             *_, count = line.split(b' ')
             count = int(count)
-            self.assertLessEqual(count, i*5)
+            self.assertLessEqual(count, i*10)
-            self.assertGreaterEqual(count, i*5-2)
+            self.assertGreaterEqual(count, i*10-4)
 
 
 # Py_GIL_DISABLED requires mimalloc (not malloc)

Lib/test/test_class.py

@@ -2,7 +2,6 @@
 
 import unittest
 
-
 testmeths = [
 
 # Binary operations
@@ -789,5 +788,81 @@ class ClassTests(unittest.TestCase):
         self.assertEqual(calls, 100)
 
 
+from _testinternalcapi import has_inline_values
+
+Py_TPFLAGS_MANAGED_DICT = (1 << 2)
+
+class Plain:
+    pass
+
+
+class WithAttrs:
+
+    def __init__(self):
+        self.a = 1
+        self.b = 2
+        self.c = 3
+        self.d = 4
+
+
+class TestInlineValues(unittest.TestCase):
+
+    def test_flags(self):
+        self.assertEqual(Plain.__flags__ & Py_TPFLAGS_MANAGED_DICT, Py_TPFLAGS_MANAGED_DICT)
+        self.assertEqual(WithAttrs.__flags__ & Py_TPFLAGS_MANAGED_DICT, Py_TPFLAGS_MANAGED_DICT)
+
+    def test_has_inline_values(self):
+        c = Plain()
+        self.assertTrue(has_inline_values(c))
+        del c.__dict__
+        self.assertFalse(has_inline_values(c))
+
+    def test_instances(self):
+        self.assertTrue(has_inline_values(Plain()))
+        self.assertTrue(has_inline_values(WithAttrs()))
+
+    def test_inspect_dict(self):
+        for cls in (Plain, WithAttrs):
+            c = cls()
+            c.__dict__
+            self.assertTrue(has_inline_values(c))
+
+    def test_update_dict(self):
+        d = { "e": 5, "f": 6 }
+        for cls in (Plain, WithAttrs):
+            c = cls()
+            c.__dict__.update(d)
+            self.assertTrue(has_inline_values(c))
+
+    @staticmethod
+    def set_100(obj):
+        for i in range(100):
+            setattr(obj, f"a{i}", i)
+
+    def check_100(self, obj):
+        for i in range(100):
+            self.assertEqual(getattr(obj, f"a{i}"), i)
+
+    def test_many_attributes(self):
+        class C: pass
+        c = C()
+        self.assertTrue(has_inline_values(c))
+        self.set_100(c)
+        self.assertFalse(has_inline_values(c))
+        self.check_100(c)
+        c = C()
+        self.assertTrue(has_inline_values(c))
+
+    def test_many_attributes_with_dict(self):
+        class C: pass
+        c = C()
+        d = c.__dict__
+        self.assertTrue(has_inline_values(c))
+        self.set_100(c)
+        self.assertFalse(has_inline_values(c))
+        self.check_100(c)
+
+
+
 if __name__ == '__main__':
     unittest.main()

Lib/test/test_opcache.py

@@ -1044,20 +1044,13 @@ class TestInstanceDict(unittest.TestCase):
         c.a = 1
         c.b = 2
         c.__dict__
-        self.assertIs(
+        self.assertEqual(c.__dict__, {"a":1, "b": 2})
-            _testinternalcapi.get_object_dict_values(c),
-            None
-        )
 
     def test_dict_dematerialization(self):
         c = C()
         c.a = 1
         c.b = 2
         c.__dict__
-        self.assertIs(
-            _testinternalcapi.get_object_dict_values(c),
-            None
-        )
         for _ in range(100):
             c.a
         self.assertEqual(
@@ -1072,10 +1065,6 @@ class TestInstanceDict(unittest.TestCase):
         d = c.__dict__
         for _ in range(100):
             c.a
-        self.assertIs(
-            _testinternalcapi.get_object_dict_values(c),
-            None
-        )
         self.assertIs(c.__dict__, d)
 
     def test_dict_dematerialization_copy(self):

Misc/NEWS.d/next/Core and Builtins/2024-02-24-03-39-09.gh-issue-115776.THJXqg.rst

@@ -0,0 +1,4 @@
+The array of values, the ``PyDictValues`` struct is now embedded in the
+object during allocation. This provides better performance in the common
+case, and does not degrade as much when the object's ``__dict__`` is
+materialized.

Modules/_testbuffer.c

@@ -2820,6 +2820,9 @@ static int
 _testbuffer_exec(PyObject *mod)
 {
     Py_SET_TYPE(&NDArray_Type, &PyType_Type);
+    if (PyType_Ready(&NDArray_Type)) {
+        return -1;
+    }
     if (PyModule_AddType(mod, &NDArray_Type) < 0) {
         return -1;
     }

Modules/_testcapimodule.c

@@ -3869,7 +3869,9 @@ PyInit__testcapi(void)
         return NULL;
 
     Py_SET_TYPE(&_HashInheritanceTester_Type, &PyType_Type);
-
+    if (PyType_Ready(&_HashInheritanceTester_Type) < 0) {
+        return NULL;
+    }
     if (PyType_Ready(&matmulType) < 0)
         return NULL;
     Py_INCREF(&matmulType);

Modules/_testinternalcapi.c

@@ -15,7 +15,7 @@
 #include "pycore_ceval.h"         // _PyEval_AddPendingCall()
 #include "pycore_compile.h"       // _PyCompile_CodeGen()
 #include "pycore_context.h"       // _PyContext_NewHamtForTests()
-#include "pycore_dict.h"          // _PyDictOrValues_GetValues()
+#include "pycore_dict.h"          // _PyManagedDictPointer_GetValues()
 #include "pycore_fileutils.h"     // _Py_normpath()
 #include "pycore_frame.h"         // _PyInterpreterFrame
 #include "pycore_gc.h"            // PyGC_Head
@@ -1297,14 +1297,13 @@ static PyObject *
 get_object_dict_values(PyObject *self, PyObject *obj)
 {
     PyTypeObject *type = Py_TYPE(obj);
-    if (!_PyType_HasFeature(type, Py_TPFLAGS_MANAGED_DICT)) {
+    if (!_PyType_HasFeature(type, Py_TPFLAGS_INLINE_VALUES)) {
         Py_RETURN_NONE;
     }
-    PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(obj);
+    PyDictValues *values = _PyObject_InlineValues(obj);
-    if (!_PyDictOrValues_IsValues(dorv)) {
+    if (!values->valid) {
         Py_RETURN_NONE;
     }
-    PyDictValues *values = _PyDictOrValues_GetValues(dorv);
     PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys;
     assert(keys != NULL);
     int size = (int)keys->dk_nentries;
@@ -1784,6 +1783,16 @@ get_py_thread_id(PyObject *self, PyObject *Py_UNUSED(ignored))
 }
 #endif
 
+static PyObject *
+has_inline_values(PyObject *self, PyObject *obj)
+{
+    if ((Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES) &&
+        _PyObject_InlineValues(obj)->valid) {
+        Py_RETURN_TRUE;
+    }
+    Py_RETURN_FALSE;
+}
+
 static PyMethodDef module_functions[] = {
     {"get_configs", get_configs, METH_NOARGS},
     {"get_recursion_depth", get_recursion_depth, METH_NOARGS},
@@ -1857,6 +1866,7 @@ static PyMethodDef module_functions[] = {
     _TESTINTERNALCAPI_TEST_LONG_NUMBITS_METHODDEF
     {"get_rare_event_counters", get_rare_event_counters, METH_NOARGS},
     {"reset_rare_event_counters", reset_rare_event_counters, METH_NOARGS},
+    {"has_inline_values", has_inline_values, METH_O},
 #ifdef Py_GIL_DISABLED
     {"py_thread_id", get_py_thread_id, METH_NOARGS},
 #endif

Objects/dictobject.c

@@ -361,6 +361,10 @@ static int
 dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_value,
                     PyObject **result, int incref_result);
 
+#ifndef NDEBUG
+static int _PyObject_InlineValuesConsistencyCheck(PyObject *obj);
+#endif
+
 #include "clinic/dictobject.c.h"
 
 
@@ -624,8 +628,9 @@ static inline int
 get_index_from_order(PyDictObject *mp, Py_ssize_t i)
 {
     assert(mp->ma_used <= SHARED_KEYS_MAX_SIZE);
-    assert(i < (((char *)mp->ma_values)[-2]));
+    assert(i < mp->ma_values->size);
-    return ((char *)mp->ma_values)[-3-i];
+    uint8_t *array = get_insertion_order_array(mp->ma_values);
+    return array[i];
 }
 
 #ifdef DEBUG_PYDICT
@@ -672,6 +677,10 @@ _PyDict_CheckConsistency(PyObject *op, int check_content)
     else {
         CHECK(keys->dk_kind == DICT_KEYS_SPLIT);
         CHECK(mp->ma_used <= SHARED_KEYS_MAX_SIZE);
+        if (mp->ma_values->embedded) {
+            CHECK(mp->ma_values->embedded == 1);
+            CHECK(mp->ma_values->valid == 1);
+        }
     }
 
     if (check_content) {
@@ -821,33 +830,44 @@ free_keys_object(PyDictKeysObject *keys, bool use_qsbr)
     PyMem_Free(keys);
 }
 
+static size_t
+values_size_from_count(size_t count)
+{
+    assert(count >= 1);
+    size_t suffix_size = _Py_SIZE_ROUND_UP(count, sizeof(PyObject *));
+    assert(suffix_size < 128);
+    assert(suffix_size % sizeof(PyObject *) == 0);
+    return (count + 1) * sizeof(PyObject *) + suffix_size;
+}
+
+#define CACHED_KEYS(tp) (((PyHeapTypeObject*)tp)->ht_cached_keys)
+
 static inline PyDictValues*
 new_values(size_t size)
 {
-    assert(size >= 1);
+    size_t n = values_size_from_count(size);
-    size_t prefix_size = _Py_SIZE_ROUND_UP(size+2, sizeof(PyObject *));
+    PyDictValues *res = (PyDictValues *)PyMem_Malloc(n);
-    assert(prefix_size < 256);
+    if (res == NULL) {
-    size_t n = prefix_size + size * sizeof(PyObject *);
-    uint8_t *mem = PyMem_Malloc(n);
-    if (mem == NULL) {
         return NULL;
     }
-    assert(prefix_size % sizeof(PyObject *) == 0);
+    res->embedded = 0;
-    mem[prefix_size-1] = (uint8_t)prefix_size;
+    res->size = 0;
-    return (PyDictValues*)(mem + prefix_size);
+    assert(size < 256);
+    res->capacity = (uint8_t)size;
+    return res;
 }
 
 static inline void
 free_values(PyDictValues *values, bool use_qsbr)
 {
-    int prefix_size = DICT_VALUES_SIZE(values);
+    assert(values->embedded == 0);
 #ifdef Py_GIL_DISABLED
     if (use_qsbr) {
-        _PyMem_FreeDelayed(((char *)values)-prefix_size);
+        _PyMem_FreeDelayed(values);
         return;
     }
 #endif
-    PyMem_Free(((char *)values)-prefix_size);
+    PyMem_Free(values);
 }
 
 /* Consumes a reference to the keys object */
@@ -887,24 +907,6 @@ new_dict(PyInterpreterState *interp,
     return (PyObject *)mp;
 }
 
-static inline size_t
-shared_keys_usable_size(PyDictKeysObject *keys)
-{
-#ifdef Py_GIL_DISABLED
-    // dk_usable will decrease for each instance that is created and each
-    // value that is added.  dk_nentries will increase for each value that
-    // is added.  We want to always return the right value or larger.
-    // We therefore increase dk_nentries first and we decrease dk_usable
-    // second, and conversely here we read dk_usable first and dk_entries
-    // second (to avoid the case where we read entries before the increment
-    // and read usable after the decrement)
-    return (size_t)(_Py_atomic_load_ssize_acquire(&keys->dk_usable) +
-                    _Py_atomic_load_ssize_acquire(&keys->dk_nentries));
-#else
-    return (size_t)keys->dk_nentries + (size_t)keys->dk_usable;
-#endif
-}
-
 /* Consumes a reference to the keys object */
 static PyObject *
 new_dict_with_shared_keys(PyInterpreterState *interp, PyDictKeysObject *keys)
@@ -915,7 +917,6 @@ new_dict_with_shared_keys(PyInterpreterState *interp, PyDictKeysObject *keys)
         dictkeys_decref(interp, keys, false);
         return PyErr_NoMemory();
     }
-    ((char *)values)[-2] = 0;
     for (size_t i = 0; i < size; i++) {
         values->values[i] = NULL;
     }
@@ -1419,7 +1420,7 @@ _Py_dict_lookup_threadsafe(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyOb
                 if (values == NULL)
                     goto read_failed;
 
-                uint8_t capacity = _Py_atomic_load_uint8_relaxed(&DICT_VALUES_SIZE(values));
+                uint8_t capacity = _Py_atomic_load_uint8_relaxed(&values->capacity);
                 if (ix >= (Py_ssize_t)capacity)
                     goto read_failed;
 
@@ -1525,6 +1526,7 @@ _PyDict_MaybeUntrack(PyObject *op)
         return;
 
     mp = (PyDictObject *) op;
+    ASSERT_CONSISTENT(mp);
     numentries = mp->ma_keys->dk_nentries;
     if (_PyDict_HasSplitTable(mp)) {
         for (i = 0; i < numentries; i++) {
@@ -1945,8 +1947,15 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp,
         set_keys(mp, newkeys);
         dictkeys_decref(interp, oldkeys, IS_DICT_SHARED(mp));
         set_values(mp, NULL);
+        if (oldvalues->embedded) {
+            assert(oldvalues->embedded == 1);
+            assert(oldvalues->valid == 1);
+            oldvalues->valid = 0;
+        }
+        else {
             free_values(oldvalues, IS_DICT_SHARED(mp));
         }
+    }
     else {  // oldkeys is combined.
         if (oldkeys->dk_kind == DICT_KEYS_GENERAL) {
             // generic -> generic
@@ -2464,17 +2473,19 @@ _PyDict_SetItem_KnownHash(PyObject *op, PyObject *key, PyObject *value,
 static void
 delete_index_from_values(PyDictValues *values, Py_ssize_t ix)
 {
-    uint8_t *size_ptr = ((uint8_t *)values)-2;
+    uint8_t *array = get_insertion_order_array(values);
-    int size = *size_ptr;
+    int size = values->size;
+    assert(size <= values->capacity);
     int i;
-    for (i = 1; size_ptr[-i] != ix; i++) {
+    for (i = 0; array[i] != ix; i++) {
-        assert(i <= size);
+        assert(i < size);
     }
-    assert(i <= size);
+    assert(i < size);
+    size--;
     for (; i < size; i++) {
-        size_ptr[-i] = size_ptr[-i-1];
+        array[i] = array[i+1];
     }
-    *size_ptr = size -1;
+    values->size = size;
 }
 
 static int
@@ -2669,10 +2680,12 @@ clear_lock_held(PyObject *op)
     mp->ma_version_tag = new_version;
     /* ...then clear the keys and values */
     if (oldvalues != NULL) {
+        if (!oldvalues->embedded) {
             n = oldkeys->dk_nentries;
             for (i = 0; i < n; i++)
                 Py_CLEAR(oldvalues->values[i]);
             free_values(oldvalues, IS_DICT_SHARED(mp));
+        }
         dictkeys_decref(interp, oldkeys, false);
     }
     else {
@@ -3059,10 +3072,12 @@ dict_dealloc(PyObject *self)
     PyObject_GC_UnTrack(mp);
     Py_TRASHCAN_BEGIN(mp, dict_dealloc)
     if (values != NULL) {
+        if (values->embedded == 0) {
             for (i = 0, n = mp->ma_keys->dk_nentries; i < n; i++) {
                 Py_XDECREF(values->values[i]);
             }
             free_values(values, false);
+        }
         dictkeys_decref(interp, keys, false);
     }
     else if (keys != NULL) {
@@ -3595,10 +3610,12 @@ dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *othe
         PyDictKeysObject *okeys = other->ma_keys;
 
         // If other is clean, combined, and just allocated, just clone it.
-        if (other->ma_values == NULL &&
+        if (mp->ma_values == NULL &&
+            other->ma_values == NULL &&
             other->ma_used == okeys->dk_nentries &&
             (DK_LOG_SIZE(okeys) == PyDict_LOG_MINSIZE ||
-                    USABLE_FRACTION(DK_SIZE(okeys)/2) < other->ma_used)) {
+             USABLE_FRACTION(DK_SIZE(okeys)/2) < other->ma_used)
+        ) {
             uint64_t new_version = _PyDict_NotifyEvent(
                     interp, PyDict_EVENT_CLONED, mp, (PyObject *)other, NULL);
             PyDictKeysObject *keys = clone_combined_dict_keys(other);
@@ -3608,11 +3625,6 @@ dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *othe
             ensure_shared_on_resize(mp);
             dictkeys_decref(interp, mp->ma_keys, IS_DICT_SHARED(mp));
             mp->ma_keys = keys;
-            if (_PyDict_HasSplitTable(mp)) {
-                free_values(mp->ma_values, IS_DICT_SHARED(mp));
-                mp->ma_values = NULL;
-            }
-
             mp->ma_used = other->ma_used;
             mp->ma_version_tag = new_version;
             ASSERT_CONSISTENT(mp);
@@ -3816,6 +3828,27 @@ dict_copy_impl(PyDictObject *self)
     return PyDict_Copy((PyObject *)self);
 }
 
+/* Copies the values, but does not change the reference
+ * counts of the objects in the array. */
+static PyDictValues *
+copy_values(PyDictValues *values)
+{
+    PyDictValues *newvalues = new_values(values->capacity);
+    if (newvalues == NULL) {
+        PyErr_NoMemory();
+        return NULL;
+    }
+    newvalues->size = values->size;
+    uint8_t *values_order = get_insertion_order_array(values);
+    uint8_t *new_values_order = get_insertion_order_array(newvalues);
+    memcpy(new_values_order, values_order, values->capacity);
+    for (int i = 0; i < values->capacity; i++) {
+        newvalues->values[i] = values->values[i];
+    }
+    assert(newvalues->embedded == 0);
+    return newvalues;
+}
+
 static PyObject *
 copy_lock_held(PyObject *o)
 {
@@ -3833,26 +3866,23 @@ copy_lock_held(PyObject *o)
 
     if (_PyDict_HasSplitTable(mp)) {
         PyDictObject *split_copy;
-        size_t size = shared_keys_usable_size(mp->ma_keys);
+        PyDictValues *newvalues = copy_values(mp->ma_values);
-        PyDictValues *newvalues = new_values(size);
+        if (newvalues == NULL) {
-        if (newvalues == NULL)
             return PyErr_NoMemory();
+        }
         split_copy = PyObject_GC_New(PyDictObject, &PyDict_Type);
         if (split_copy == NULL) {
             free_values(newvalues, false);
             return NULL;
         }
-        size_t prefix_size = ((uint8_t *)newvalues)[-1];
+        for (size_t i = 0; i < newvalues->capacity; i++) {
-        memcpy(((char *)newvalues)-prefix_size, ((char *)mp->ma_values)-prefix_size, prefix_size-1);
+            Py_XINCREF(newvalues->values[i]);
+        }
         split_copy->ma_values = newvalues;
         split_copy->ma_keys = mp->ma_keys;
         split_copy->ma_used = mp->ma_used;
         split_copy->ma_version_tag = DICT_NEXT_VERSION(interp);
         dictkeys_incref(mp->ma_keys);
-        for (size_t i = 0; i < size; i++) {
-            PyObject *value = mp->ma_values->values[i];
-            split_copy->ma_values->values[i] = Py_XNewRef(value);
-        }
         if (_PyObject_GC_IS_TRACKED(mp))
             _PyObject_GC_TRACK(split_copy);
         return (PyObject *)split_copy;
@@ -4406,10 +4436,12 @@ dict_traverse(PyObject *op, visitproc visit, void *arg)
 
     if (DK_IS_UNICODE(keys)) {
         if (_PyDict_HasSplitTable(mp)) {
+            if (!mp->ma_values->embedded) {
                 for (i = 0; i < n; i++) {
                     Py_VISIT(mp->ma_values->values[i]);
                 }
             }
+        }
         else {
             PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys);
             for (i = 0; i < n; i++) {
@@ -5296,12 +5328,6 @@ acquire_key_value(PyObject **key_loc, PyObject *value, PyObject **value_loc,
     return 0;
 }
 
-static Py_ssize_t
-load_values_used_size(PyDictValues *values)
-{
-    return (Py_ssize_t)_Py_atomic_load_uint8(&DICT_VALUES_USED_SIZE(values));
-}
-
 static int
 dictiter_iternext_threadsafe(PyDictObject *d, PyObject *self,
                              PyObject **out_key, PyObject **out_value)
@@ -5330,7 +5356,7 @@ dictiter_iternext_threadsafe(PyDictObject *d, PyObject *self,
             goto concurrent_modification;
         }
 
-        Py_ssize_t used = load_values_used_size(values);
+        Py_ssize_t used = (Py_ssize_t)_Py_atomic_load_uint8(&values->size);
         if (i >= used) {
             goto fail;
         }
@@ -6539,15 +6565,15 @@ _PyDict_NewKeysForClass(void)
     return keys;
 }
 
-#define CACHED_KEYS(tp) (((PyHeapTypeObject*)tp)->ht_cached_keys)
+void
-
-int
 _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp)
 {
     assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE);
+    assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES);
     assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT);
     PyDictKeysObject *keys = CACHED_KEYS(tp);
     assert(keys != NULL);
+    OBJECT_STAT_INC(inline_values);
 #ifdef Py_GIL_DISABLED
     Py_ssize_t usable = _Py_atomic_load_ssize_relaxed(&keys->dk_usable);
     if (usable > 1) {
@@ -6563,49 +6589,19 @@ _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp)
     }
 #endif
     size_t size = shared_keys_usable_size(keys);
-    PyDictValues *values = new_values(size);
+    PyDictValues *values = _PyObject_InlineValues(obj);
-    if (values == NULL) {
+    assert(size < 256);
-        PyErr_NoMemory();
+    values->capacity = (uint8_t)size;
-        return -1;
+    values->size = 0;
-    }
+    values->embedded = 1;
-    assert(((uint8_t *)values)[-1] >= (size + 2));
+    values->valid = 1;
-    ((uint8_t *)values)[-2] = 0;
     for (size_t i = 0; i < size; i++) {
         values->values[i] = NULL;
     }
-    _PyDictOrValues_SetValues(_PyObject_DictOrValuesPointer(obj), values);
+    _PyObject_ManagedDictPointer(obj)->dict = NULL;
-    return 0;
 }
 
-int
+static PyDictObject *
-_PyObject_InitializeDict(PyObject *obj)
-{
-    PyInterpreterState *interp = _PyInterpreterState_GET();
-    PyTypeObject *tp = Py_TYPE(obj);
-    if (tp->tp_dictoffset == 0) {
-        return 0;
-    }
-    if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
-        OBJECT_STAT_INC(new_values);
-        return _PyObject_InitInlineValues(obj, tp);
-    }
-    PyObject *dict;
-    if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && CACHED_KEYS(tp)) {
-        dictkeys_incref(CACHED_KEYS(tp));
-        dict = new_dict_with_shared_keys(interp, CACHED_KEYS(tp));
-    }
-    else {
-        dict = PyDict_New();
-    }
-    if (dict == NULL) {
-        return -1;
-    }
-    PyObject **dictptr = _PyObject_ComputedDictPointer(obj);
-    *dictptr = dict;
-    return 0;
-}
-
-static PyObject *
 make_dict_from_instance_attributes(PyInterpreterState *interp,
                                    PyDictKeysObject *keys, PyDictValues *values)
 {
@@ -6620,56 +6616,24 @@ make_dict_from_instance_attributes(PyInterpreterState *interp,
             track += _PyObject_GC_MAY_BE_TRACKED(val);
         }
     }
-    PyObject *res = new_dict(interp, keys, values, used, 0);
+    PyDictObject *res = (PyDictObject *)new_dict(interp, keys, values, used, 0);
     if (track && res) {
         _PyObject_GC_TRACK(res);
     }
     return res;
 }
 
-PyObject *
+PyDictObject *
-_PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values)
+_PyObject_MakeDictFromInstanceAttributes(PyObject *obj)
 {
+    PyDictValues *values = _PyObject_InlineValues(obj);
     PyInterpreterState *interp = _PyInterpreterState_GET();
     PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj));
     OBJECT_STAT_INC(dict_materialized_on_request);
     return make_dict_from_instance_attributes(interp, keys, values);
 }
 
-// Return true if the dict was dematerialized, false otherwise.
-bool
-_PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv)
-{
-    assert(_PyObject_DictOrValuesPointer(obj) == dorv);
-    assert(!_PyDictOrValues_IsValues(*dorv));
-    PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(*dorv);
-    if (dict == NULL) {
-        return false;
-    }
-    // It's likely that this dict still shares its keys (if it was materialized
-    // on request and not heavily modified):
-    if (!PyDict_CheckExact(dict)) {
-        return false;
-    }
-    assert(_PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_HEAPTYPE));
-    if (dict->ma_keys != CACHED_KEYS(Py_TYPE(obj)) ||
-        !has_unique_reference((PyObject *)dict))
-    {
-        return false;
-    }
-    ensure_shared_on_resize(dict);
 
-    assert(dict->ma_values);
-    // We have an opportunity to do something *really* cool: dematerialize it!
-    _PyDictKeys_DecRef(dict->ma_keys);
-    _PyDictOrValues_SetValues(dorv, dict->ma_values);
-    OBJECT_STAT_INC(dict_dematerialized);
-    // Don't try this at home, kids:
-    dict->ma_keys = NULL;
-    dict->ma_values = NULL;
-    Py_DECREF(dict);
-    return true;
-}
 
 int
 _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values,
@@ -6679,7 +6643,7 @@ _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values,
     PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj));
     assert(keys != NULL);
     assert(values != NULL);
-    assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
+    assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
     Py_ssize_t ix = DKIX_EMPTY;
     if (PyUnicode_CheckExact(name)) {
         Py_hash_t hash = unicode_get_hash(name);
@@ -6717,18 +6681,21 @@ _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values,
         }
 #endif
     }
+    PyDictObject *dict = _PyObject_ManagedDictPointer(obj)->dict;
     if (ix == DKIX_EMPTY) {
-        PyObject *dict = make_dict_from_instance_attributes(
+        if (dict == NULL) {
+            dict = make_dict_from_instance_attributes(
                     interp, keys, values);
             if (dict == NULL) {
                 return -1;
             }
-        _PyObject_DictOrValuesPointer(obj)->dict = dict;
+            _PyObject_ManagedDictPointer(obj)->dict = (PyDictObject *)dict;
+        }
         if (value == NULL) {
-            return PyDict_DelItem(dict, name);
+            return PyDict_DelItem((PyObject *)dict, name);
         }
         else {
-            return PyDict_SetItem(dict, name, value);
+            return PyDict_SetItem((PyObject *)dict, name, value);
         }
     }
     PyObject *old_value = values->values[ix];
@@ -6741,10 +6708,18 @@ _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values,
             return -1;
         }
         _PyDictValues_AddToInsertionOrder(values, ix);
+        if (dict) {
+            assert(dict->ma_values == values);
+            dict->ma_used++;
+        }
     }
     else {
         if (value == NULL) {
             delete_index_from_values(values, ix);
+            if (dict) {
+                assert(dict->ma_values == values);
+                dict->ma_used--;
+            }
         }
         Py_DECREF(old_value);
     }
@@ -6760,9 +6735,9 @@ _PyObject_ManagedDictValidityCheck(PyObject *obj)
 {
     PyTypeObject *tp = Py_TYPE(obj);
     CHECK(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT);
-    PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj);
+    PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(obj);
-    if (_PyDictOrValues_IsValues(*dorv_ptr)) {
+    if (_PyManagedDictPointer_IsValues(*managed_dict)) {
-        PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr);
+        PyDictValues *values = _PyManagedDictPointer_GetValues(*managed_dict);
         int size = ((uint8_t *)values)[-2];
         int count = 0;
         PyDictKeysObject *keys = CACHED_KEYS(tp);
@@ -6774,8 +6749,8 @@ _PyObject_ManagedDictValidityCheck(PyObject *obj)
         CHECK(size == count);
     }
     else {
-        if (dorv_ptr->dict != NULL) {
+        if (managed_dict->dict != NULL) {
-            CHECK(PyDict_Check(dorv_ptr->dict));
+            CHECK(PyDict_Check(managed_dict->dict));
         }
     }
     return 1;
@@ -6804,23 +6779,27 @@ _PyObject_IsInstanceDictEmpty(PyObject *obj)
     if (tp->tp_dictoffset == 0) {
         return 1;
     }
-    PyObject *dict;
+    PyDictObject *dict;
-    if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
+    if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) {
-        PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(obj);
+        PyDictValues *values = _PyObject_InlineValues(obj);
-        if (_PyDictOrValues_IsValues(dorv)) {
+        if (values->valid) {
             PyDictKeysObject *keys = CACHED_KEYS(tp);
             for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) {
-                if (_PyDictOrValues_GetValues(dorv)->values[i] != NULL) {
+                if (values->values[i] != NULL) {
                     return 0;
                 }
             }
             return 1;
         }
-        dict = _PyDictOrValues_GetDict(dorv);
+        dict = _PyObject_ManagedDictPointer(obj)->dict;
+    }
+    else if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
+        PyManagedDictPointer* managed_dict = _PyObject_ManagedDictPointer(obj);
+        dict = managed_dict->dict;
     }
     else {
         PyObject **dictptr = _PyObject_ComputedDictPointer(obj);
-        dict = *dictptr;
+        dict = (PyDictObject *)*dictptr;
     }
     if (dict == NULL) {
         return 1;
@@ -6828,23 +6807,6 @@ _PyObject_IsInstanceDictEmpty(PyObject *obj)
     return ((PyDictObject *)dict)->ma_used == 0;
 }
 
-void
-_PyObject_FreeInstanceAttributes(PyObject *self)
-{
-    PyTypeObject *tp = Py_TYPE(self);
-    assert(Py_TYPE(self)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
-    PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(self);
-    if (!_PyDictOrValues_IsValues(dorv)) {
-        return;
-    }
-    PyDictValues *values = _PyDictOrValues_GetValues(dorv);
-    PyDictKeysObject *keys = CACHED_KEYS(tp);
-    for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) {
-        Py_XDECREF(values->values[i]);
-    }
-    free_values(values, IS_DICT_SHARED((PyDictObject*)self));
-}
-
 int
 PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg)
 {
@@ -6852,74 +6814,101 @@ PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg)
     if((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) {
         return 0;
     }
-    assert(tp->tp_dictoffset);
+    if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) {
-    PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(obj);
+        PyDictValues *values = _PyObject_InlineValues(obj);
-    if (_PyDictOrValues_IsValues(dorv)) {
+        if (values->valid) {
-        PyDictValues *values = _PyDictOrValues_GetValues(dorv);
+            for (Py_ssize_t i = 0; i < values->capacity; i++) {
-        PyDictKeysObject *keys = CACHED_KEYS(tp);
-        for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) {
                 Py_VISIT(values->values[i]);
             }
+            return 0;
         }
-    else {
-        PyObject *dict = _PyDictOrValues_GetDict(dorv);
-        Py_VISIT(dict);
     }
+    Py_VISIT(_PyObject_ManagedDictPointer(obj)->dict);
     return 0;
 }
 
 void
 PyObject_ClearManagedDict(PyObject *obj)
 {
+    assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
+    assert(_PyObject_InlineValuesConsistencyCheck(obj));
     PyTypeObject *tp = Py_TYPE(obj);
-    if((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) {
+    if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) {
-        return;
+        PyDictObject *dict = _PyObject_ManagedDictPointer(obj)->dict;
-    }
-    PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj);
-    if (_PyDictOrValues_IsValues(*dorv_ptr)) {
-        PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr);
-        PyDictKeysObject *keys = CACHED_KEYS(tp);
-        for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) {
-            Py_CLEAR(values->values[i]);
-        }
-        dorv_ptr->dict = NULL;
-        free_values(values, IS_DICT_SHARED((PyDictObject*)obj));
-    }
-    else {
-        PyObject *dict = dorv_ptr->dict;
         if (dict) {
-            dorv_ptr->dict = NULL;
+            _PyDict_DetachFromObject(dict, obj);
+            _PyObject_ManagedDictPointer(obj)->dict = NULL;
             Py_DECREF(dict);
         }
+        else {
+            PyDictValues *values = _PyObject_InlineValues(obj);
+            if (values->valid) {
+                for (Py_ssize_t i = 0; i < values->capacity; i++) {
+                    Py_CLEAR(values->values[i]);
                 }
+                values->valid = 0;
+            }
+        }
+    }
+    else {
+        Py_CLEAR(_PyObject_ManagedDictPointer(obj)->dict);
+    }
+    assert(_PyObject_InlineValuesConsistencyCheck(obj));
+}
+
+int
+_PyDict_DetachFromObject(PyDictObject *mp, PyObject *obj)
+{
+    assert(_PyObject_ManagedDictPointer(obj)->dict == mp);
+    assert(_PyObject_InlineValuesConsistencyCheck(obj));
+    if (mp->ma_values == NULL || mp->ma_values != _PyObject_InlineValues(obj)) {
+        return 0;
+    }
+    assert(mp->ma_values->embedded == 1);
+    assert(mp->ma_values->valid == 1);
+    assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
+    Py_BEGIN_CRITICAL_SECTION(mp);
+    mp->ma_values = copy_values(mp->ma_values);
+    _PyObject_InlineValues(obj)->valid = 0;
+    Py_END_CRITICAL_SECTION();
+    if (mp->ma_values == NULL) {
+        return -1;
+    }
+    assert(_PyObject_InlineValuesConsistencyCheck(obj));
+    ASSERT_CONSISTENT(mp);
+    return 0;
 }
 
 PyObject *
 PyObject_GenericGetDict(PyObject *obj, void *context)
 {
-    PyObject *dict;
     PyInterpreterState *interp = _PyInterpreterState_GET();
     PyTypeObject *tp = Py_TYPE(obj);
     if (_PyType_HasFeature(tp, Py_TPFLAGS_MANAGED_DICT)) {
-        PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj);
+        PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(obj);
-        if (_PyDictOrValues_IsValues(*dorv_ptr)) {
+        PyDictObject *dict = managed_dict->dict;
-            PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr);
+        if (dict == NULL &&
+            (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) &&
+            _PyObject_InlineValues(obj)->valid
+        ) {
+            PyDictValues *values = _PyObject_InlineValues(obj);
             OBJECT_STAT_INC(dict_materialized_on_request);
             dict = make_dict_from_instance_attributes(
                     interp, CACHED_KEYS(tp), values);
             if (dict != NULL) {
-                dorv_ptr->dict = dict;
+                managed_dict->dict = (PyDictObject *)dict;
             }
         }
         else {
-            dict = _PyDictOrValues_GetDict(*dorv_ptr);
+            dict = managed_dict->dict;
             if (dict == NULL) {
                 dictkeys_incref(CACHED_KEYS(tp));
                 OBJECT_STAT_INC(dict_materialized_on_request);
-                dict = new_dict_with_shared_keys(interp, CACHED_KEYS(tp));
+                dict = (PyDictObject *)new_dict_with_shared_keys(interp, CACHED_KEYS(tp));
-                dorv_ptr->dict = dict;
+                managed_dict->dict = (PyDictObject *)dict;
             }
         }
+        return Py_XNewRef((PyObject *)dict);
     }
     else {
         PyObject **dictptr = _PyObject_ComputedDictPointer(obj);
@@ -6928,7 +6917,7 @@ PyObject_GenericGetDict(PyObject *obj, void *context)
                             "This object has no __dict__");
             return NULL;
         }
-        dict = *dictptr;
+        PyObject *dict = *dictptr;
         if (dict == NULL) {
             PyTypeObject *tp = Py_TYPE(obj);
             if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && CACHED_KEYS(tp)) {
@@ -6940,9 +6929,9 @@ PyObject_GenericGetDict(PyObject *obj, void *context)
                 *dictptr = dict = PyDict_New();
             }
         }
-    }
         return Py_XNewRef(dict);
     }
+}
 
 int
 _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr,
@@ -6958,7 +6947,7 @@ _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr,
         assert(dictptr != NULL);
         dict = *dictptr;
         if (dict == NULL) {
-            assert(!_PyType_HasFeature(tp, Py_TPFLAGS_MANAGED_DICT));
+            assert(!_PyType_HasFeature(tp, Py_TPFLAGS_INLINE_VALUES));
             dictkeys_incref(cached);
             dict = new_dict_with_shared_keys(interp, cached);
             if (dict == NULL)
@@ -7118,3 +7107,24 @@ _PyDict_SendEvent(int watcher_bits,
         watcher_bits >>= 1;
     }
 }
+
+#ifndef NDEBUG
+static int
+_PyObject_InlineValuesConsistencyCheck(PyObject *obj)
+{
+    if ((Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES) == 0) {
+        return 1;
+    }
+    assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
+    PyDictObject *dict = (PyDictObject *)_PyObject_ManagedDictPointer(obj)->dict;
+    if (dict == NULL) {
+        return 1;
+    }
+    if (dict->ma_values == _PyObject_InlineValues(obj) ||
+        _PyObject_InlineValues(obj)->valid == 0) {
+        return 1;
+    }
+    assert(0);
+    return 0;
+}
+#endif

Objects/object.c

@@ -1396,16 +1396,16 @@ _PyObject_GetDictPtr(PyObject *obj)
     if ((Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) {
         return _PyObject_ComputedDictPointer(obj);
     }
-    PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj);
+    PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(obj);
-    if (_PyDictOrValues_IsValues(*dorv_ptr)) {
+    if (managed_dict->dict == NULL && Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES) {
-        PyObject *dict = _PyObject_MakeDictFromInstanceAttributes(obj, _PyDictOrValues_GetValues(*dorv_ptr));
+        PyDictObject *dict = (PyDictObject *)_PyObject_MakeDictFromInstanceAttributes(obj);
         if (dict == NULL) {
             PyErr_Clear();
             return NULL;
         }
-        dorv_ptr->dict = dict;
+        managed_dict->dict = dict;
     }
-    return &dorv_ptr->dict;
+    return (PyObject **)&managed_dict->dict;
 }
 
 PyObject *
@@ -1474,10 +1474,8 @@ _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method)
         }
     }
     PyObject *dict;
-    if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) {
+    if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) && _PyObject_InlineValues(obj)->valid) {
-        PyDictOrValues* dorv_ptr = _PyObject_DictOrValuesPointer(obj);
+        PyDictValues *values = _PyObject_InlineValues(obj);
-        if (_PyDictOrValues_IsValues(*dorv_ptr)) {
-            PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr);
         PyObject *attr = _PyObject_GetInstanceAttribute(obj, values, name);
         if (attr != NULL) {
             *method = attr;
@@ -1486,9 +1484,9 @@ _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method)
         }
         dict = NULL;
     }
-        else {
+    else if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) {
-            dict = dorv_ptr->dict;
+        PyManagedDictPointer* managed_dict = _PyObject_ManagedDictPointer(obj);
-        }
+        dict = (PyObject *)managed_dict->dict;
     }
     else {
         PyObject **dictptr = _PyObject_ComputedDictPointer(obj);
@@ -1581,10 +1579,8 @@ _PyObject_GenericGetAttrWithDict(PyObject *obj, PyObject *name,
         }
     }
     if (dict == NULL) {
-        if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) {
+        if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) && _PyObject_InlineValues(obj)->valid) {
-            PyDictOrValues* dorv_ptr = _PyObject_DictOrValuesPointer(obj);
+            PyDictValues *values = _PyObject_InlineValues(obj);
-            if (_PyDictOrValues_IsValues(*dorv_ptr)) {
-                PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr);
             if (PyUnicode_CheckExact(name)) {
                 res = _PyObject_GetInstanceAttribute(obj, values, name);
                 if (res != NULL) {
@@ -1592,17 +1588,17 @@ _PyObject_GenericGetAttrWithDict(PyObject *obj, PyObject *name,
                 }
             }
             else {
-                    dict = _PyObject_MakeDictFromInstanceAttributes(obj, values);
+                dict = (PyObject *)_PyObject_MakeDictFromInstanceAttributes(obj);
                 if (dict == NULL) {
                     res = NULL;
                     goto done;
                 }
-                    dorv_ptr->dict = dict;
+                _PyObject_ManagedDictPointer(obj)->dict = (PyDictObject *)dict;
             }
         }
-            else {
+        else if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) {
-                dict = _PyDictOrValues_GetDict(*dorv_ptr);
+            PyManagedDictPointer* managed_dict = _PyObject_ManagedDictPointer(obj);
-            }
+            dict = (PyObject *)managed_dict->dict;
         }
         else {
             PyObject **dictptr = _PyObject_ComputedDictPointer(obj);
@@ -1697,22 +1693,14 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name,
 
     if (dict == NULL) {
         PyObject **dictptr;
-        if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) {
+        if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) && _PyObject_InlineValues(obj)->valid) {
-            PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj);
-            if (_PyDictOrValues_IsValues(*dorv_ptr)) {
             res = _PyObject_StoreInstanceAttribute(
-                    obj, _PyDictOrValues_GetValues(*dorv_ptr), name, value);
+                    obj, _PyObject_InlineValues(obj), name, value);
-                goto error_check;
-            }
-            dictptr = &dorv_ptr->dict;
-            if (*dictptr == NULL) {
-                if (_PyObject_InitInlineValues(obj, tp) < 0) {
-                    goto done;
-                }
-                res = _PyObject_StoreInstanceAttribute(
-                    obj, _PyDictOrValues_GetValues(*dorv_ptr), name, value);
             goto error_check;
         }
+        else if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) {
+            PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(obj);
+            dictptr = (PyObject **)&managed_dict->dict;
         }
         else {
             dictptr = _PyObject_ComputedDictPointer(obj);
@@ -1783,9 +1771,9 @@ PyObject_GenericSetDict(PyObject *obj, PyObject *value, void *context)
 {
     PyObject **dictptr = _PyObject_GetDictPtr(obj);
     if (dictptr == NULL) {
-        if (_PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_MANAGED_DICT) &&
+        if (_PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_INLINE_VALUES) &&
-            _PyDictOrValues_IsValues(*_PyObject_DictOrValuesPointer(obj)))
+            _PyObject_ManagedDictPointer(obj)->dict == NULL
-        {
+        ) {
             /* Was unable to convert to dict */
             PyErr_NoMemory();
         }

Objects/object_layout.md

@@ -16,25 +16,23 @@ Since the introduction of the cycle GC, there has also been a pre-header.
 Before 3.11, this pre-header was two words in size.
 It should be considered opaque to all code except the cycle GC.
 
-## 3.11 pre-header
+### 3.13
 
-In 3.11 the pre-header was extended to include pointers to the VM managed ``__dict__``.
+In 3.13, the values array is embedded into the object, so there is no
-The reason for moving the ``__dict__`` to the pre-header is that it allows
+need for a values pointer (it is just a fixed offset into the object).
-faster access, as it is at a fixed offset, and it also allows object's
+So the pre-header is these two fields:
-dictionaries to be lazily created when the ``__dict__`` attribute is
-specifically asked for.
 
-In the 3.11 the non-GC part of the pre-header consists of two pointers:
+* weakreflist
+* dict_pointer
 
-* dict
+If the object has no physical dictionary, then the ``dict_pointer``
-* values
+is set to `NULL`.
 
-The values pointer refers to the ``PyDictValues`` array which holds the
-values of the objects's attributes.
-Should the dictionary be needed, then ``values`` is set to ``NULL``
-and the ``dict`` field points to the dictionary.
 
-## 3.12 pre-header
+<details>
+<summary> 3.12 </summary>
+
+### 3.12
 
 In 3.12, the pointer to the list of weak references is added to the
 pre-header. In order to make space for it, the ``dict`` and ``values``
@@ -51,9 +49,38 @@ has its low bit set to zero, and points to the dictionary.
 The untagged form is chosen for the dictionary pointer, rather than
 the values pointer, to enable the (legacy) C-API function
 `_PyObject_GetDictPtr(PyObject *obj)` to work.
+</details>
 
+<details>
+<summary> 3.11 </summary>
 
-## Layout of a "normal" Python object in 3.12:
+### 3.11
+
+In 3.11 the pre-header was extended to include pointers to the VM managed ``__dict__``.
+The reason for moving the ``__dict__`` to the pre-header is that it allows
+faster access, as it is at a fixed offset, and it also allows object's
+dictionaries to be lazily created when the ``__dict__`` attribute is
+specifically asked for.
+
+In the 3.11 the non-GC part of the pre-header consists of two pointers:
+
+* dict
+* values
+
+The values pointer refers to the ``PyDictValues`` array which holds the
+values of the objects's attributes.
+Should the dictionary be needed, then ``values`` is set to ``NULL``
+and the ``dict`` field points to the dictionary.
+</details>
+
+## Layout of a "normal" Python object
+
+A "normal" Python object is one that doesn't inherit from a builtin
+class, doesn't have slots.
+
+### 3.13
+
+In 3.13 the values are embedded into the object, as follows:
 
 * weakreflist
 * dict_or_values
@@ -61,9 +88,39 @@ the values pointer, to enable the (legacy) C-API function
 * GC 2
 * ob_refcnt
 * ob_type
+* Inlined values:
+  * Flags
+  * values 0
+  * values 1
+  * ...
+  * Insertion order bytes
 
-For a "normal" Python object, one that doesn't inherit from a builtin
+This has all the advantages of the layout used in 3.12, plus:
-class or have slots, the header and pre-header form the entire object.
+* Access to values is even faster as there is one less load
+* Fast access is mostly maintained when the `__dict__` is materialized
+
+![Layout of "normal" object in 3.13](./object_layout_313.png)
+
+For objects with opaque parts defined by a C extension,
+the layout is much the same as for 3.12
+
+![Layout of "full" object in 3.13](./object_layout_full_313.png)
+
+
+<details>
+<summary> 3.12 </summary>
+
+### 3.12:
+
+In 3.12, the header and pre-header form the entire object for "normal"
+Python objects:
+
+* weakreflist
+* dict_or_values
+* GC 1
+* GC 2
+* ob_refcnt
+* ob_type
 
 ![Layout of "normal" object in 3.12](./object_layout_312.png)
 
@@ -79,4 +136,6 @@ The full layout object, with an opaque part defined by a C extension,
 and `__slots__` looks like this:
 
 ![Layout of "full" object in 3.12](./object_layout_full_312.png)
+</details>
+
 

Objects/object_layout_312.gv

@@ -20,6 +20,7 @@ digraph ideal {
         shape = none
         label = <<table border="0" cellspacing="0">
                     <tr><td><b>values</b></td></tr>
+                    <tr><td border="1">Insertion order</td></tr>
                     <tr><td port="0" border="1">values[0]</td></tr>
                     <tr><td border="1">values[1]</td></tr>
                     <tr><td border="1">...</td></tr>

Objects/object_layout_313.gv

@@ -0,0 +1,45 @@
+digraph ideal {
+
+    rankdir = "LR"
+
+
+    object [
+        shape = none
+        label = <<table border="0" cellspacing="0">
+                    <tr><td><b>object</b></td></tr>
+                    <tr><td port="w" border="1">weakrefs</td></tr>
+                    <tr><td port="dv" border="1">dict pointer</td></tr>
+                    <tr><td border="1" >GC info 0</td></tr>
+                    <tr><td border="1" >GC info 1</td></tr>
+                    <tr><td port="r" border="1" >refcount</td></tr>
+                    <tr><td port="h" border="1" >__class__</td></tr>
+                    <tr><td border="1">values flags</td></tr>
+                    <tr><td port="0" border="1">values[0]</td></tr>
+                    <tr><td border="1">values[1]</td></tr>
+                    <tr><td border="1">...</td></tr>
+                    <tr><td border="1">Insertion order</td></tr>
+                </table>>
+
+    ]
+
+    class [ 
+        shape = none
+        label = <<table border="0" cellspacing="0">
+                    <tr><td><b>class</b></td></tr>
+                    <tr><td port="head" bgcolor="lightgreen" border="1">...</td></tr>
+                    <tr><td border="1" bgcolor="lightgreen">dict_offset</td></tr>
+                    <tr><td border="1" bgcolor="lightgreen">...</td></tr>
+                    <tr><td port="k" border="1" bgcolor="lightgreen">cached_keys</td></tr>
+                </table>>
+    ]
+
+    keys [label = "dictionary keys"; fillcolor="lightgreen"; style="filled"]
+    NULL [ label = " NULL"; shape="plain"]
+    object:w ->  NULL
+    object:h -> class:head
+    object:dv -> NULL
+    class:k -> keys
+
+    oop [ label = "pointer"; shape="plain"]
+    oop -> object:r
+}

Objects/object_layout_full_313.gv

@@ -0,0 +1,25 @@
+digraph ideal {
+
+    rankdir = "LR"
+
+
+    object [
+        shape = none
+        label = <<table border="0" cellspacing="0">
+                    <tr><td><b>object</b></td></tr>
+                    <tr><td port="w" border="1">weakrefs</td></tr>
+                    <tr><td port="dv" border="1">dict pointer</td></tr>
+                    <tr><td border="1" >GC info 0</td></tr>
+                    <tr><td border="1" >GC info 1</td></tr>
+                    <tr><td port="r" border="1" >refcount</td></tr>
+                    <tr><td port="h" border="1" >__class__</td></tr>
+                    <tr><td border="1">opaque (extension) data </td></tr>
+                    <tr><td border="1">...</td></tr>
+                    <tr><td border="1">__slot__ 0</td></tr>
+                    <tr><td border="1">...</td></tr>
+                </table>>
+    ]
+
+    oop [ label = "pointer"; shape="plain"]
+    oop -> object:r
+}

Objects/typeobject.c

@@ -1861,7 +1861,7 @@ type_call(PyObject *self, PyObject *args, PyObject *kwds)
 PyObject *
 _PyType_NewManagedObject(PyTypeObject *type)
 {
-    assert(type->tp_flags & Py_TPFLAGS_MANAGED_DICT);
+    assert(type->tp_flags & Py_TPFLAGS_INLINE_VALUES);
     assert(_PyType_IS_GC(type));
     assert(type->tp_new == PyBaseObject_Type.tp_new);
     assert(type->tp_alloc == PyType_GenericAlloc);
@@ -1870,11 +1870,6 @@ _PyType_NewManagedObject(PyTypeObject *type)
     if (obj == NULL) {
         return PyErr_NoMemory();
     }
-    _PyObject_DictOrValuesPointer(obj)->dict = NULL;
-    if (_PyObject_InitInlineValues(obj, type)) {
-        Py_DECREF(obj);
-        return NULL;
-    }
     return obj;
 }
 
@@ -1888,9 +1883,13 @@ _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems)
      * flag to indicate when that is safe) it does not seem worth the memory
      * savings. An example type that doesn't need the +1 is a subclass of
      * tuple. See GH-100659 and GH-81381. */
-    const size_t size = _PyObject_VAR_SIZE(type, nitems+1);
+    size_t size = _PyObject_VAR_SIZE(type, nitems+1);
 
     const size_t presize = _PyType_PreHeaderSize(type);
+    if (type->tp_flags & Py_TPFLAGS_INLINE_VALUES) {
+        assert(type->tp_itemsize == 0);
+        size += _PyInlineValuesSize(type);
+    }
     char *alloc = _PyObject_MallocWithType(type, size + presize);
     if (alloc  == NULL) {
         return PyErr_NoMemory();
@@ -1911,6 +1910,9 @@ _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems)
     else {
         _PyObject_InitVar((PyVarObject *)obj, type, nitems);
     }
+    if (type->tp_flags & Py_TPFLAGS_INLINE_VALUES) {
+        _PyObject_InitInlineValues(obj, type);
+    }
     return obj;
 }
 
@@ -2060,6 +2062,10 @@ subtype_clear(PyObject *self)
         if ((base->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) {
             PyObject_ClearManagedDict(self);
         }
+        else {
+            assert((base->tp_flags & Py_TPFLAGS_INLINE_VALUES) ==
+                   (type->tp_flags & Py_TPFLAGS_INLINE_VALUES));
+        }
     }
     else if (type->tp_dictoffset != base->tp_dictoffset) {
         PyObject **dictptr = _PyObject_ComputedDictPointer(self);
@@ -2210,14 +2216,7 @@ subtype_dealloc(PyObject *self)
 
     /* If we added a dict, DECREF it, or free inline values. */
     if (type->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
-        PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(self);
+        PyObject_ClearManagedDict(self);
-        if (_PyDictOrValues_IsValues(*dorv_ptr)) {
-            _PyObject_FreeInstanceAttributes(self);
-        }
-        else {
-            Py_XDECREF(_PyDictOrValues_GetDict(*dorv_ptr));
-        }
-        dorv_ptr->values = NULL;
     }
     else if (type->tp_dictoffset && !base->tp_dictoffset) {
         PyObject **dictptr = _PyObject_ComputedDictPointer(self);
@@ -3161,19 +3160,26 @@ subtype_setdict(PyObject *obj, PyObject *value, void *context)
         return func(descr, obj, value);
     }
     /* Almost like PyObject_GenericSetDict, but allow __dict__ to be deleted. */
-    dictptr = _PyObject_GetDictPtr(obj);
-    if (dictptr == NULL) {
-        PyErr_SetString(PyExc_AttributeError,
-                        "This object has no __dict__");
-        return -1;
-    }
     if (value != NULL && !PyDict_Check(value)) {
         PyErr_Format(PyExc_TypeError,
                      "__dict__ must be set to a dictionary, "
                      "not a '%.200s'", Py_TYPE(value)->tp_name);
         return -1;
     }
-    Py_XSETREF(*dictptr, Py_XNewRef(value));
+    if (Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
+        PyObject_ClearManagedDict(obj);
+        _PyObject_ManagedDictPointer(obj)->dict = (PyDictObject *)Py_XNewRef(value);
+    }
+    else {
+        dictptr = _PyObject_ComputedDictPointer(obj);
+        if (dictptr == NULL) {
+            PyErr_SetString(PyExc_AttributeError,
+                            "This object has no __dict__");
+            return -1;
+        }
+        Py_CLEAR(*dictptr);
+        *dictptr = Py_XNewRef(value);
+    }
     return 0;
 }
 
@@ -5849,10 +5855,6 @@ object_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
     if (obj == NULL) {
         return NULL;
     }
-    if (_PyObject_InitializeDict(obj)) {
-        Py_DECREF(obj);
-        return NULL;
-    }
     return obj;
 }
 
@@ -6036,6 +6038,11 @@ compatible_for_assignment(PyTypeObject* oldto, PyTypeObject* newto, const char*
          !same_slots_added(newbase, oldbase))) {
         goto differs;
     }
+    if ((oldto->tp_flags & Py_TPFLAGS_INLINE_VALUES) !=
+        ((newto->tp_flags & Py_TPFLAGS_INLINE_VALUES)))
+    {
+        goto differs;
+    }
     /* The above does not check for the preheader */
     if ((oldto->tp_flags & Py_TPFLAGS_PREHEADER) ==
         ((newto->tp_flags & Py_TPFLAGS_PREHEADER)))
@@ -6137,15 +6144,19 @@ object_set_class(PyObject *self, PyObject *value, void *closure)
     if (compatible_for_assignment(oldto, newto, "__class__")) {
         /* Changing the class will change the implicit dict keys,
          * so we must materialize the dictionary first. */
-        assert((oldto->tp_flags & Py_TPFLAGS_PREHEADER) == (newto->tp_flags & Py_TPFLAGS_PREHEADER));
+        if (oldto->tp_flags & Py_TPFLAGS_INLINE_VALUES) {
-        _PyObject_GetDictPtr(self);
+            PyDictObject *dict = _PyObject_ManagedDictPointer(self)->dict;
-        if (oldto->tp_flags & Py_TPFLAGS_MANAGED_DICT &&
+            if (dict == NULL) {
-            _PyDictOrValues_IsValues(*_PyObject_DictOrValuesPointer(self)))
+                dict = (PyDictObject *)_PyObject_MakeDictFromInstanceAttributes(self);
-        {
+                if (dict == NULL) {
-            /* Was unable to convert to dict */
-            PyErr_NoMemory();
                     return -1;
                 }
+                _PyObject_ManagedDictPointer(self)->dict = dict;
+            }
+            if (_PyDict_DetachFromObject(dict, self)) {
+                return -1;
+            }
+        }
         if (newto->tp_flags & Py_TPFLAGS_HEAPTYPE) {
             Py_INCREF(newto);
         }
@@ -7774,6 +7785,9 @@ type_ready_managed_dict(PyTypeObject *type)
             return -1;
         }
     }
+    if (type->tp_itemsize == 0 && type->tp_basicsize == sizeof(PyObject)) {
+        type->tp_flags |= Py_TPFLAGS_INLINE_VALUES;
+    }
     return 0;
 }
 
@@ -7901,6 +7915,8 @@ PyType_Ready(PyTypeObject *type)
     /* Historically, all static types were immutable. See bpo-43908 */
     if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) {
         type->tp_flags |= Py_TPFLAGS_IMMUTABLETYPE;
+        /* Static types must be immortal */
+        _Py_SetImmortalUntracked((PyObject *)type);
     }
 
     int res;

Python/bytecodes.c

@@ -1897,14 +1897,12 @@ dummy_func(
 
         op(_CHECK_MANAGED_OBJECT_HAS_VALUES, (owner -- owner)) {
             assert(Py_TYPE(owner)->tp_dictoffset < 0);
-            assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
+            assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
-            PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner);
+            DEOPT_IF(!_PyObject_InlineValues(owner)->valid);
-            DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv));
         }
 
         split op(_LOAD_ATTR_INSTANCE_VALUE, (index/1, owner -- attr, null if (oparg & 1))) {
-            PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
+            attr = _PyObject_InlineValues(owner)->values[index];
-            attr = _PyDictOrValues_GetValues(dorv)->values[index];
             DEOPT_IF(attr == NULL);
             STAT_INC(LOAD_ATTR, hit);
             Py_INCREF(attr);
@@ -1947,16 +1945,15 @@ dummy_func(
 
         op(_CHECK_ATTR_WITH_HINT, (owner -- owner)) {
             assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
-            PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
+            PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner);
-            DEOPT_IF(_PyDictOrValues_IsValues(dorv));
+            PyDictObject *dict = managed_dict->dict;
-            PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv);
             DEOPT_IF(dict == NULL);
             assert(PyDict_CheckExact((PyObject *)dict));
         }
 
         op(_LOAD_ATTR_WITH_HINT, (hint/1, owner -- attr, null if (oparg & 1))) {
-            PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
+            PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner);
-            PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv);
+            PyDictObject *dict = managed_dict->dict;
             DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries);
             PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
             if (DK_IS_UNICODE(dict->ma_keys)) {
@@ -2070,16 +2067,17 @@ dummy_func(
             DISPATCH_INLINED(new_frame);
         }
 
-        op(_GUARD_DORV_VALUES, (owner -- owner)) {
+        op(_GUARD_DORV_NO_DICT, (owner -- owner)) {
-            assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
+            assert(Py_TYPE(owner)->tp_dictoffset < 0);
-            PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
+            assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
-            DEOPT_IF(!_PyDictOrValues_IsValues(dorv));
+            DEOPT_IF(_PyObject_ManagedDictPointer(owner)->dict);
+            DEOPT_IF(_PyObject_InlineValues(owner)->valid == 0);
         }
 
         op(_STORE_ATTR_INSTANCE_VALUE, (index/1, value, owner --)) {
-            PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
             STAT_INC(STORE_ATTR, hit);
-            PyDictValues *values = _PyDictOrValues_GetValues(dorv);
+            assert(_PyObject_ManagedDictPointer(owner)->dict == NULL);
+            PyDictValues *values = _PyObject_InlineValues(owner);
             PyObject *old_value = values->values[index];
             values->values[index] = value;
             if (old_value == NULL) {
@@ -2094,7 +2092,7 @@ dummy_func(
         macro(STORE_ATTR_INSTANCE_VALUE) =
             unused/1 +
             _GUARD_TYPE_VERSION +
-            _GUARD_DORV_VALUES +
+            _GUARD_DORV_NO_DICT +
             _STORE_ATTR_INSTANCE_VALUE;
 
         inst(STORE_ATTR_WITH_HINT, (unused/1, type_version/2, hint/1, value, owner --)) {
@@ -2102,9 +2100,8 @@ dummy_func(
             assert(type_version != 0);
             DEOPT_IF(tp->tp_version_tag != type_version);
             assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT);
-            PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
+            PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner);
-            DEOPT_IF(_PyDictOrValues_IsValues(dorv));
+            PyDictObject *dict = managed_dict->dict;
-            PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv);
             DEOPT_IF(dict == NULL);
             assert(PyDict_CheckExact((PyObject *)dict));
             PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
@@ -2898,9 +2895,8 @@ dummy_func(
         }
 
         op(_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, (owner -- owner)) {
-            assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
+            assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
-            PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner);
+            DEOPT_IF(!_PyObject_InlineValues(owner)->valid);
-            DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv));
         }
 
         op(_GUARD_KEYS_VERSION, (keys_version/2, owner -- owner)) {
@@ -2972,10 +2968,9 @@ dummy_func(
             unused/2 +
             _LOAD_ATTR_NONDESCRIPTOR_NO_DICT;
 
-        op(_CHECK_ATTR_METHOD_LAZY_DICT, (owner -- owner)) {
+        op(_CHECK_ATTR_METHOD_LAZY_DICT, (dictoffset/1, owner -- owner)) {
-            Py_ssize_t dictoffset = Py_TYPE(owner)->tp_dictoffset;
+            char *ptr = ((char *)owner) + MANAGED_DICT_OFFSET + dictoffset;
-            assert(dictoffset > 0);
+            PyObject *dict = *(PyObject **)ptr;
-            PyObject *dict = *(PyObject **)((char *)owner + dictoffset);
             /* This object has a __dict__, just not yet created */
             DEOPT_IF(dict != NULL);
         }
@@ -2993,7 +2988,7 @@ dummy_func(
             unused/1 +
             _GUARD_TYPE_VERSION +
             _CHECK_ATTR_METHOD_LAZY_DICT +
-            unused/2 +
+            unused/1 +
             _LOAD_ATTR_METHOD_LAZY_DICT;
 
         inst(INSTRUMENTED_CALL, (unused/3 -- )) {
@@ -3294,6 +3289,7 @@ dummy_func(
             DEOPT_IF(!PyType_Check(callable));
             PyTypeObject *tp = (PyTypeObject *)callable;
             DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version));
+            assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES);
             PyHeapTypeObject *cls = (PyHeapTypeObject *)callable;
             PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init;
             PyCodeObject *code = (PyCodeObject *)init->func_code;

Python/executor_cases.c.h

@@ -1753,9 +1753,8 @@
             PyObject *owner;
             owner = stack_pointer[-1];
             assert(Py_TYPE(owner)->tp_dictoffset < 0);
-            assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
+            assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
-            PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner);
+            if (!_PyObject_InlineValues(owner)->valid) JUMP_TO_JUMP_TARGET();
-            if (!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)) JUMP_TO_JUMP_TARGET();
             break;
         }
 
@@ -1766,8 +1765,7 @@
             (void)null;
             owner = stack_pointer[-1];
             uint16_t index = (uint16_t)CURRENT_OPERAND();
-            PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
+            attr = _PyObject_InlineValues(owner)->values[index];
-            attr = _PyDictOrValues_GetValues(dorv)->values[index];
             if (attr == NULL) JUMP_TO_JUMP_TARGET();
             STAT_INC(LOAD_ATTR, hit);
             Py_INCREF(attr);
@@ -1784,8 +1782,7 @@
             (void)null;
             owner = stack_pointer[-1];
             uint16_t index = (uint16_t)CURRENT_OPERAND();
-            PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
+            attr = _PyObject_InlineValues(owner)->values[index];
-            attr = _PyDictOrValues_GetValues(dorv)->values[index];
             if (attr == NULL) JUMP_TO_JUMP_TARGET();
             STAT_INC(LOAD_ATTR, hit);
             Py_INCREF(attr);
@@ -1837,9 +1834,8 @@
             PyObject *owner;
             owner = stack_pointer[-1];
             assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
-            PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
+            PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner);
-            if (_PyDictOrValues_IsValues(dorv)) JUMP_TO_JUMP_TARGET();
+            PyDictObject *dict = managed_dict->dict;
-            PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv);
             if (dict == NULL) JUMP_TO_JUMP_TARGET();
             assert(PyDict_CheckExact((PyObject *)dict));
             break;
@@ -1852,8 +1848,8 @@
             oparg = CURRENT_OPARG();
             owner = stack_pointer[-1];
             uint16_t hint = (uint16_t)CURRENT_OPERAND();
-            PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
+            PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner);
-            PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv);
+            PyDictObject *dict = managed_dict->dict;
             if (hint >= (size_t)dict->ma_keys->dk_nentries) JUMP_TO_JUMP_TARGET();
             PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
             if (DK_IS_UNICODE(dict->ma_keys)) {
@@ -1967,12 +1963,13 @@
 
         /* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
 
-        case _GUARD_DORV_VALUES: {
+        case _GUARD_DORV_NO_DICT: {
             PyObject *owner;
             owner = stack_pointer[-1];
-            assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
+            assert(Py_TYPE(owner)->tp_dictoffset < 0);
-            PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
+            assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
-            if (!_PyDictOrValues_IsValues(dorv)) JUMP_TO_JUMP_TARGET();
+            if (_PyObject_ManagedDictPointer(owner)->dict) JUMP_TO_JUMP_TARGET();
+            if (_PyObject_InlineValues(owner)->valid == 0) JUMP_TO_JUMP_TARGET();
             break;
         }
 
@@ -1982,9 +1979,9 @@
             owner = stack_pointer[-1];
             value = stack_pointer[-2];
             uint16_t index = (uint16_t)CURRENT_OPERAND();
-            PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
             STAT_INC(STORE_ATTR, hit);
-            PyDictValues *values = _PyDictOrValues_GetValues(dorv);
+            assert(_PyObject_ManagedDictPointer(owner)->dict == NULL);
+            PyDictValues *values = _PyObject_InlineValues(owner);
             PyObject *old_value = values->values[index];
             values->values[index] = value;
             if (old_value == NULL) {
@@ -2568,9 +2565,8 @@
         case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: {
             PyObject *owner;
             owner = stack_pointer[-1];
-            assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
+            assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
-            PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner);
+            if (!_PyObject_InlineValues(owner)->valid) JUMP_TO_JUMP_TARGET();
-            if (!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)) JUMP_TO_JUMP_TARGET();
             break;
         }
 
@@ -2658,9 +2654,9 @@
         case _CHECK_ATTR_METHOD_LAZY_DICT: {
             PyObject *owner;
             owner = stack_pointer[-1];
-            Py_ssize_t dictoffset = Py_TYPE(owner)->tp_dictoffset;
+            uint16_t dictoffset = (uint16_t)CURRENT_OPERAND();
-            assert(dictoffset > 0);
+            char *ptr = ((char *)owner) + MANAGED_DICT_OFFSET + dictoffset;
-            PyObject *dict = *(PyObject **)((char *)owner + dictoffset);
+            PyObject *dict = *(PyObject **)ptr;
             /* This object has a __dict__, just not yet created */
             if (dict != NULL) JUMP_TO_JUMP_TARGET();
             break;

Python/gc.c

@@ -2031,11 +2031,16 @@ gc_alloc(PyTypeObject *tp, size_t basicsize, size_t presize)
     return op;
 }
 
+
 PyObject *
 _PyObject_GC_New(PyTypeObject *tp)
 {
     size_t presize = _PyType_PreHeaderSize(tp);
-    PyObject *op = gc_alloc(tp, _PyObject_SIZE(tp), presize);
+    size_t size = _PyObject_SIZE(tp);
+    if (_PyType_HasFeature(tp, Py_TPFLAGS_INLINE_VALUES)) {
+        size += _PyInlineValuesSize(tp);
+    }
+    PyObject *op = gc_alloc(tp, size, presize);
     if (op == NULL) {
         return NULL;
     }

Python/gc_free_threading.c

@@ -1639,7 +1639,11 @@ PyObject *
 _PyObject_GC_New(PyTypeObject *tp)
 {
     size_t presize = _PyType_PreHeaderSize(tp);
-    PyObject *op = gc_alloc(tp, _PyObject_SIZE(tp), presize);
+    size_t size = _PyObject_SIZE(tp);
+    if (_PyType_HasFeature(tp, Py_TPFLAGS_INLINE_VALUES)) {
+        size += _PyInlineValuesSize(tp);
+    }
+    PyObject *op = gc_alloc(tp, size, presize);
     if (op == NULL) {
         return NULL;
     }

Python/generated_cases.c.h

@@ -870,6 +870,7 @@
             DEOPT_IF(!PyType_Check(callable), CALL);
             PyTypeObject *tp = (PyTypeObject *)callable;
             DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version), CALL);
+            assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES);
             PyHeapTypeObject *cls = (PyHeapTypeObject *)callable;
             PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init;
             PyCodeObject *code = (PyCodeObject *)init->func_code;
@@ -3680,15 +3681,13 @@
             // _CHECK_MANAGED_OBJECT_HAS_VALUES
             {
                 assert(Py_TYPE(owner)->tp_dictoffset < 0);
-                assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
+                assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
-                PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner);
+                DEOPT_IF(!_PyObject_InlineValues(owner)->valid, LOAD_ATTR);
-                DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR);
             }
             // _LOAD_ATTR_INSTANCE_VALUE
             {
                 uint16_t index = read_u16(&this_instr[4].cache);
-                PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
+                attr = _PyObject_InlineValues(owner)->values[index];
-                attr = _PyDictOrValues_GetValues(dorv)->values[index];
                 DEOPT_IF(attr == NULL, LOAD_ATTR);
                 STAT_INC(LOAD_ATTR, hit);
                 Py_INCREF(attr);
@@ -3721,13 +3720,13 @@
             }
             // _CHECK_ATTR_METHOD_LAZY_DICT
             {
-                Py_ssize_t dictoffset = Py_TYPE(owner)->tp_dictoffset;
+                uint16_t dictoffset = read_u16(&this_instr[4].cache);
-                assert(dictoffset > 0);
+                char *ptr = ((char *)owner) + MANAGED_DICT_OFFSET + dictoffset;
-                PyObject *dict = *(PyObject **)((char *)owner + dictoffset);
+                PyObject *dict = *(PyObject **)ptr;
                 /* This object has a __dict__, just not yet created */
                 DEOPT_IF(dict != NULL, LOAD_ATTR);
             }
-            /* Skip 2 cache entries */
+            /* Skip 1 cache entry */
             // _LOAD_ATTR_METHOD_LAZY_DICT
             {
                 PyObject *descr = read_obj(&this_instr[6].cache);
@@ -3798,9 +3797,8 @@
             }
             // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT
             {
-                assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
+                assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
-                PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner);
+                DEOPT_IF(!_PyObject_InlineValues(owner)->valid, LOAD_ATTR);
-                DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR);
             }
             // _GUARD_KEYS_VERSION
             {
@@ -3914,9 +3912,8 @@
             }
             // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT
             {
-                assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
+                assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
-                PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner);
+                DEOPT_IF(!_PyObject_InlineValues(owner)->valid, LOAD_ATTR);
-                DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR);
             }
             // _GUARD_KEYS_VERSION
             {
@@ -4026,17 +4023,16 @@
             // _CHECK_ATTR_WITH_HINT
             {
                 assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
-                PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
+                PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner);
-                DEOPT_IF(_PyDictOrValues_IsValues(dorv), LOAD_ATTR);
+                PyDictObject *dict = managed_dict->dict;
-                PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv);
                 DEOPT_IF(dict == NULL, LOAD_ATTR);
                 assert(PyDict_CheckExact((PyObject *)dict));
             }
             // _LOAD_ATTR_WITH_HINT
             {
                 uint16_t hint = read_u16(&this_instr[4].cache);
-                PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
+                PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner);
-                PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv);
+                PyDictObject *dict = managed_dict->dict;
                 DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR);
                 PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
                 if (DK_IS_UNICODE(dict->ma_keys)) {
@@ -5315,19 +5311,20 @@
                 assert(type_version != 0);
                 DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
             }
-            // _GUARD_DORV_VALUES
+            // _GUARD_DORV_NO_DICT
             {
-                assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
+                assert(Py_TYPE(owner)->tp_dictoffset < 0);
-                PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
+                assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
-                DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR);
+                DEOPT_IF(_PyObject_ManagedDictPointer(owner)->dict, STORE_ATTR);
+                DEOPT_IF(_PyObject_InlineValues(owner)->valid == 0, STORE_ATTR);
             }
             // _STORE_ATTR_INSTANCE_VALUE
             value = stack_pointer[-2];
             {
                 uint16_t index = read_u16(&this_instr[4].cache);
-                PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
                 STAT_INC(STORE_ATTR, hit);
-                PyDictValues *values = _PyDictOrValues_GetValues(dorv);
+                assert(_PyObject_ManagedDictPointer(owner)->dict == NULL);
+                PyDictValues *values = _PyObject_InlineValues(owner);
                 PyObject *old_value = values->values[index];
                 values->values[index] = value;
                 if (old_value == NULL) {
@@ -5389,9 +5386,8 @@
             assert(type_version != 0);
             DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
             assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT);
-            PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
+            PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner);
-            DEOPT_IF(_PyDictOrValues_IsValues(dorv), STORE_ATTR);
+            PyDictObject *dict = managed_dict->dict;
-            PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv);
             DEOPT_IF(dict == NULL, STORE_ATTR);
             assert(PyDict_CheckExact((PyObject *)dict));
             PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);

Python/optimizer_cases.c.h

@@ -1104,7 +1104,7 @@
 
         /* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 */
 
-        case _GUARD_DORV_VALUES: {
+        case _GUARD_DORV_NO_DICT: {
             break;
         }
 

Python/specialize.c

@@ -188,7 +188,7 @@ print_object_stats(FILE *out, ObjectStats *stats)
     fprintf(out, "Object allocations to 4 kbytes: %" PRIu64 "\n", stats->allocations4k);
     fprintf(out, "Object allocations over 4 kbytes: %" PRIu64 "\n", stats->allocations_big);
     fprintf(out, "Object frees: %" PRIu64 "\n", stats->frees);
-    fprintf(out, "Object new values: %" PRIu64 "\n", stats->new_values);
+    fprintf(out, "Object inline values: %" PRIu64 "\n", stats->inline_values);
     fprintf(out, "Object interpreter increfs: %" PRIu64 "\n", stats->interpreter_increfs);
     fprintf(out, "Object interpreter decrefs: %" PRIu64 "\n", stats->interpreter_decrefs);
     fprintf(out, "Object increfs: %" PRIu64 "\n", stats->increfs);
@@ -197,7 +197,6 @@ print_object_stats(FILE *out, ObjectStats *stats)
     fprintf(out, "Object materialize dict (new key): %" PRIu64 "\n", stats->dict_materialized_new_key);
     fprintf(out, "Object materialize dict (too big): %" PRIu64 "\n", stats->dict_materialized_too_big);
     fprintf(out, "Object materialize dict (str subclass): %" PRIu64 "\n", stats->dict_materialized_str_subclass);
-    fprintf(out, "Object dematerialize dict: %" PRIu64 "\n", stats->dict_dematerialized);
     fprintf(out, "Object method cache hits: %" PRIu64 "\n", stats->type_cache_hits);
     fprintf(out, "Object method cache misses: %" PRIu64 "\n", stats->type_cache_misses);
     fprintf(out, "Object method cache collisions: %" PRIu64 "\n", stats->type_cache_collisions);
@@ -479,12 +478,11 @@ _PyCode_Quicken(PyCodeObject *code)
 #define SPEC_FAIL_ATTR_NOT_MANAGED_DICT 18
 #define SPEC_FAIL_ATTR_NON_STRING_OR_SPLIT 19
 #define SPEC_FAIL_ATTR_MODULE_ATTR_NOT_FOUND 20
-
 #define SPEC_FAIL_ATTR_SHADOWED 21
 #define SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD 22
 #define SPEC_FAIL_ATTR_CLASS_METHOD_OBJ 23
 #define SPEC_FAIL_ATTR_OBJECT_SLOT 24
-#define SPEC_FAIL_ATTR_HAS_MANAGED_DICT 25
+
 #define SPEC_FAIL_ATTR_INSTANCE_ATTRIBUTE 26
 #define SPEC_FAIL_ATTR_METACLASS_ATTRIBUTE 27
 #define SPEC_FAIL_ATTR_PROPERTY_NOT_PY_FUNCTION 28
@@ -558,6 +556,7 @@ _PyCode_Quicken(PyCodeObject *code)
 #define SPEC_FAIL_CALL_OPERATOR_WRAPPER 29
 #define SPEC_FAIL_CALL_INIT_NOT_SIMPLE 30
 #define SPEC_FAIL_CALL_METACLASS 31
+#define SPEC_FAIL_CALL_INIT_NOT_INLINE_VALUES 32
 
 /* COMPARE_OP */
 #define SPEC_FAIL_COMPARE_OP_DIFFERENT_TYPES 12
@@ -829,11 +828,7 @@ specialize_dict_access(
         return 0;
     }
     _PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
-    PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner);
+    if (type->tp_flags & Py_TPFLAGS_INLINE_VALUES && _PyObject_InlineValues(owner)->valid) {
-    if (_PyDictOrValues_IsValues(*dorv) ||
-        _PyObject_MakeInstanceAttributesFromDict(owner, dorv))
-    {
-        // Virtual dictionary
         PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys;
         assert(PyUnicode_CheckExact(name));
         Py_ssize_t index = _PyDictKeys_StringLookup(keys, name);
@@ -850,7 +845,8 @@ specialize_dict_access(
         instr->op.code = values_op;
     }
     else {
-        PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(*dorv);
+        PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner);
+        PyDictObject *dict = managed_dict->dict;
         if (dict == NULL || !PyDict_CheckExact(dict)) {
             SPECIALIZATION_FAIL(base_op, SPEC_FAIL_NO_DICT);
             return 0;
@@ -1258,15 +1254,8 @@ PyObject *descr, DescriptorClassification kind, bool is_method)
 
     assert(descr != NULL);
     assert((is_method && kind == METHOD) || (!is_method && kind == NON_DESCRIPTOR));
-    if (owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
+    if (owner_cls->tp_flags & Py_TPFLAGS_INLINE_VALUES) {
-        PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner);
         PyDictKeysObject *keys = ((PyHeapTypeObject *)owner_cls)->ht_cached_keys;
-        if (!_PyDictOrValues_IsValues(*dorv) &&
-            !_PyObject_MakeInstanceAttributesFromDict(owner, dorv))
-        {
-            SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_HAS_MANAGED_DICT);
-            return 0;
-        }
         Py_ssize_t index = _PyDictKeys_StringLookup(keys, name);
         if (index != DKIX_EMPTY) {
             SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_SHADOWED);
@@ -1282,11 +1271,17 @@ PyObject *descr, DescriptorClassification kind, bool is_method)
         instr->op.code = is_method ? LOAD_ATTR_METHOD_WITH_VALUES : LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES;
     }
     else {
-        Py_ssize_t dictoffset = owner_cls->tp_dictoffset;
+        Py_ssize_t dictoffset;
-        if (dictoffset < 0 || dictoffset > INT16_MAX) {
+        if (owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
+            dictoffset = MANAGED_DICT_OFFSET;
+        }
+        else {
+            dictoffset = owner_cls->tp_dictoffset;
+            if (dictoffset < 0 || dictoffset > INT16_MAX + MANAGED_DICT_OFFSET) {
                 SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_RANGE);
                 return 0;
             }
+        }
         if (dictoffset == 0) {
             instr->op.code = is_method ? LOAD_ATTR_METHOD_NO_DICT : LOAD_ATTR_NONDESCRIPTOR_NO_DICT;
         }
@@ -1296,8 +1291,12 @@ PyObject *descr, DescriptorClassification kind, bool is_method)
                 SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_NOT_MANAGED_DICT);
                 return 0;
             }
-            assert(owner_cls->tp_dictoffset > 0);
+            /* Cache entries must be unsigned values, so we offset the
-            assert(owner_cls->tp_dictoffset <= INT16_MAX);
+             * dictoffset by MANAGED_DICT_OFFSET.
+             * We do the reverese offset in LOAD_ATTR_METHOD_LAZY_DICT */
+            dictoffset -= MANAGED_DICT_OFFSET;
+            assert(((uint16_t)dictoffset) == dictoffset);
+            cache->dict_offset = (uint16_t)dictoffset;
             instr->op.code = LOAD_ATTR_METHOD_LAZY_DICT;
         }
         else {
@@ -1729,8 +1728,8 @@ get_init_for_simple_managed_python_class(PyTypeObject *tp)
         SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OVERRIDDEN);
         return NULL;
     }
-    if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) {
+    if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) == 0) {
-        SPECIALIZATION_FAIL(CALL, SPEC_FAIL_NO_DICT);
+        SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_INIT_NOT_INLINE_VALUES);
         return NULL;
     }
     if (!(tp->tp_flags & Py_TPFLAGS_HEAPTYPE)) {

Tools/cases_generator/analyzer.py

@@ -359,11 +359,10 @@ def has_error_without_pop(op: parser.InstDef) -> bool:
 
 NON_ESCAPING_FUNCTIONS = (
     "Py_INCREF",
-    "_PyDictOrValues_IsValues",
+    "_PyManagedDictPointer_IsValues",
-    "_PyObject_DictOrValuesPointer",
+    "_PyObject_ManagedDictPointer",
-    "_PyDictOrValues_GetValues",
+    "_PyObject_InlineValues",
     "_PyDictValues_AddToInsertionOrder",
-    "_PyObject_MakeInstanceAttributesFromDict",
     "Py_DECREF",
     "_Py_DECREF_SPECIALIZED",
     "DECREF_INPUTS_AND_REUSE_FLOAT",

Tools/gdb/libpython.py

@@ -66,10 +66,12 @@ def _type_unsigned_short_ptr():
 def _type_unsigned_int_ptr():
     return gdb.lookup_type('unsigned int').pointer()
 
-
 def _sizeof_void_p():
     return gdb.lookup_type('void').pointer().sizeof
 
+def _sizeof_pyobject():
+    return gdb.lookup_type('PyObject').sizeof
+
 def _managed_dict_offset():
     # See pycore_object.h
     pyobj = gdb.lookup_type("PyObject")
@@ -79,6 +81,7 @@ def _managed_dict_offset():
         return -3 * _sizeof_void_p()
 
 
+Py_TPFLAGS_INLINE_VALUES     = (1 << 2)
 Py_TPFLAGS_MANAGED_DICT      = (1 << 4)
 Py_TPFLAGS_HEAPTYPE          = (1 << 9)
 Py_TPFLAGS_LONG_SUBCLASS     = (1 << 24)
@@ -493,11 +496,12 @@ class HeapTypeObjectPtr(PyObjectPtr):
         has_values =  int_from_int(typeobj.field('tp_flags')) & Py_TPFLAGS_MANAGED_DICT
         if not has_values:
             return None
-        ptr = self._gdbval.cast(_type_char_ptr()) + _managed_dict_offset()
+        obj_ptr = self._gdbval.cast(_type_char_ptr())
-        char_ptr = ptr.cast(_type_char_ptr().pointer()).dereference()
+        dict_ptr_ptr = obj_ptr + _managed_dict_offset()
-        if (int(char_ptr) & 1) == 0:
+        dict_ptr = dict_ptr_ptr.cast(_type_char_ptr().pointer()).dereference()
+        if int(dict_ptr):
             return None
-        char_ptr += 1
+        char_ptr = obj_ptr + _sizeof_pyobject()
         values_ptr = char_ptr.cast(gdb.lookup_type("PyDictValues").pointer())
         values = values_ptr['values']
         return PyKeysValuesPair(self.get_cached_keys(), values)

Tools/scripts/summarize_stats.py

@@ -394,7 +394,7 @@ class Stats:
         return result
 
     def get_object_stats(self) -> dict[str, tuple[int, int]]:
-        total_materializations = self._data.get("Object new values", 0)
+        total_materializations = self._data.get("Object inline values", 0)
         total_allocations = self._data.get("Object allocations", 0) + self._data.get(
             "Object allocations from freelist", 0
         )
@@ -1094,8 +1094,7 @@ def object_stats_section() -> Section:
         Below, "allocations" means "allocations that are not from a freelist".
         Total allocations = "Allocations from freelist" + "Allocations".
 
-        "New values" is the number of values arrays created for objects with
+        "Inline values" is the number of values arrays inlined into objects.
-        managed dicts.
 
         The cache hit/miss numbers are for the MRO cache, split into dunder and
         other names.