Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

bpo-46841: Quicken code in-place (GH-31888) 

* Moves the bytecode to the end of the corresponding PyCodeObject, and quickens it in-place.

* Removes the almost-always-unused co_varnames, co_freevars, and co_cellvars member caches

* _PyOpcode_Deopt is a new mapping from all opcodes to their un-quickened forms.

* _PyOpcode_InlineCacheEntries is renamed to _PyOpcode_Caches

* _Py_IncrementCountAndMaybeQuicken is renamed to _PyCode_Warmup

* _Py_Quicken is renamed to _PyCode_Quicken

* _co_quickened is renamed to _co_code_adaptive (and is now a read-only memoryview).

* Do not emit unused nonzero opargs anymore in the compiler.
This commit is contained in:
Brandt Bucher 2022-03-21 04:11:17 -07:00 committed by GitHub
parent 08eb754d84
commit 2bde6827ea
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 832 additions and 688 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

157
Include/cpython/code.h
View File

@ -26,91 +26,80 @@ typedef uint16_t _Py_CODEUNIT;
// Use "unsigned char" instead of "uint8_t" here to avoid illegal aliasing:
#define _Py_SET_OPCODE(word, opcode) (((unsigned char *)&(word))[0] = (opcode))
// To avoid repeating ourselves in deepfreeze.py, all PyCodeObject members are
// defined in this macro:
#define _PyCode_DEF(SIZE) { \
PyObject_VAR_HEAD \
\
/* Note only the following fields are used in hash and/or comparisons \
* \
* - co_name \
* - co_argcount \
* - co_posonlyargcount \
* - co_kwonlyargcount \
* - co_nlocals \
* - co_stacksize \
* - co_flags \
* - co_firstlineno \
* - co_consts \
* - co_names \
* - co_localsplusnames \
* This is done to preserve the name and line number for tracebacks \
* and debuggers; otherwise, constant de-duplication would collapse \
* identical functions/lambdas defined on different lines. \
*/ \
\
/* These fields are set with provided values on new code objects. */ \
\
/* The hottest fields (in the eval loop) are grouped here at the top. */ \
PyObject *co_consts; /* list (constants used) */ \
PyObject *co_names; /* list of strings (names used) */ \
PyObject *co_exceptiontable; /* Byte string encoding exception handling \
table */ \
int co_flags; /* CO_..., see below */ \
int co_warmup; /* Warmup counter for quickening */ \
\
/* The rest are not so impactful on performance. */ \
int co_argcount; /* #arguments, except *args */ \
int co_posonlyargcount; /* #positional only arguments */ \
int co_kwonlyargcount; /* #keyword only arguments */ \
int co_stacksize; /* #entries needed for evaluation stack */ \
int co_firstlineno; /* first source line number */ \
\
/* redundant values (derived from co_localsplusnames and \
co_localspluskinds) */ \
int co_nlocalsplus; /* number of local + cell + free variables \
*/ \
int co_nlocals; /* number of local variables */ \
int co_nplaincellvars; /* number of non-arg cell variables */ \
int co_ncellvars; /* total number of cell variables */ \
int co_nfreevars; /* number of free variables */ \
\
PyObject *co_localsplusnames; /* tuple mapping offsets to names */ \
PyObject *co_localspluskinds; /* Bytes mapping to local kinds (one byte \
per variable) */ \
PyObject *co_filename; /* unicode (where it was loaded from) */ \
PyObject *co_name; /* unicode (name, for reference) */ \
PyObject *co_qualname; /* unicode (qualname, for reference) */ \
PyObject *co_linetable; /* bytes (encoding addr<->lineno mapping) \
See Objects/lnotab_notes.txt for details. \
*/ \
PyObject *co_endlinetable; /* bytes object that holds end lineno for \
instructions separated across different \
lines */ \
PyObject *co_columntable; /* bytes object that holds start/end column \
offset each instruction */ \
\
PyObject *co_weakreflist; /* to support weakrefs to code objects */ \
/* Scratch space for extra data relating to the code object. \
Type is a void* to keep the format private in codeobject.c to force \
people to go through the proper APIs. */ \
void *co_extra; \
char co_code_adaptive[(SIZE)]; \
}
/* Bytecode object */
struct PyCodeObject {
PyObject_HEAD
/* Note only the following fields are used in hash and/or comparisons
*
* - co_name
* - co_argcount
* - co_posonlyargcount
* - co_kwonlyargcount
* - co_nlocals
* - co_stacksize
* - co_flags
* - co_firstlineno
* - co_code
* - co_consts
* - co_names
* - co_varnames
* - co_freevars
* - co_cellvars
*
* This is done to preserve the name and line number for tracebacks
* and debuggers; otherwise, constant de-duplication would collapse
* identical functions/lambdas defined on different lines.
*/
/* These fields are set with provided values on new code objects. */
// The hottest fields (in the eval loop) are grouped here at the top.
PyObject *co_consts; /* list (constants used) */
PyObject *co_names; /* list of strings (names used) */
_Py_CODEUNIT *co_firstinstr; /* Pointer to first instruction, used for quickening.
Unlike the other "hot" fields, this one is
actually derived from co_code. */
PyObject *co_exceptiontable; /* Byte string encoding exception handling table */
int co_flags; /* CO_..., see below */
int co_warmup; /* Warmup counter for quickening */
// The rest are not so impactful on performance.
int co_argcount; /* #arguments, except *args */
int co_posonlyargcount; /* #positional only arguments */
int co_kwonlyargcount; /* #keyword only arguments */
int co_stacksize; /* #entries needed for evaluation stack */
int co_firstlineno; /* first source line number */
PyObject *co_code; /* instruction opcodes */
PyObject *co_localsplusnames; /* tuple mapping offsets to names */
PyObject *co_localspluskinds; /* Bytes mapping to local kinds (one byte per variable) */
PyObject *co_filename; /* unicode (where it was loaded from) */
PyObject *co_name; /* unicode (name, for reference) */
PyObject *co_qualname; /* unicode (qualname, for reference) */
PyObject *co_linetable; /* bytes (encoding addr<->lineno mapping) See
Objects/lnotab_notes.txt for details. */
PyObject *co_endlinetable; /* bytes object that holds end lineno for
instructions separated across different
lines */
PyObject *co_columntable; /* bytes object that holds start/end column
offset each instruction */
/* These fields are set with computed values on new code objects. */
// redundant values (derived from co_localsplusnames and co_localspluskinds)
int co_nlocalsplus; /* number of local + cell + free variables */
int co_nlocals; /* number of local variables */
int co_nplaincellvars; /* number of non-arg cell variables */
int co_ncellvars; /* total number of cell variables */
int co_nfreevars; /* number of free variables */
// lazily-computed values
PyObject *co_varnames; /* tuple of strings (local variable names) */
PyObject *co_cellvars; /* tuple of strings (cell variable names) */
PyObject *co_freevars; /* tuple of strings (free variable names) */
/* The remaining fields are zeroed out on new code objects. */
PyObject *co_weakreflist; /* to support weakrefs to code objects */
/* Scratch space for extra data relating to the code object.
Type is a void* to keep the format private in codeobject.c to force
people to go through the proper APIs. */
void *co_extra;
/* Quickened instructions and cache, or NULL
This should be treated as opaque by all code except the specializer and
interpreter. */
_Py_CODEUNIT *co_quickened;
};
struct PyCodeObject _PyCode_DEF(1);
/* Masks for co_flags above */
#define CO_OPTIMIZED 0x0001
@ -151,6 +140,8 @@ PyAPI_DATA(PyTypeObject) PyCode_Type;
#define PyCode_Check(op) Py_IS_TYPE(op, &PyCode_Type)
#define PyCode_GetNumFree(op) ((op)->co_nfreevars)
#define _PyCode_CODE(CO) ((_Py_CODEUNIT *)(CO)->co_code_adaptive)
#define _PyCode_NBYTES(CO) (Py_SIZE(CO) * (Py_ssize_t)sizeof(_Py_CODEUNIT))
/* Public interface */
PyAPI_FUNC(PyCodeObject *) PyCode_New(

17
Include/internal/pycore_code.h
View File

@ -92,30 +92,22 @@ typedef struct {
#define INLINE_CACHE_ENTRIES_STORE_SUBSCR CACHE_ENTRIES(_PyStoreSubscrCache)
/* Maximum size of code to quicken, in code units. */
#define MAX_SIZE_TO_QUICKEN 10000
#define QUICKENING_WARMUP_DELAY 8
/* We want to compare to zero for efficiency, so we offset values accordingly */
#define QUICKENING_INITIAL_WARMUP_VALUE (-QUICKENING_WARMUP_DELAY)
#define QUICKENING_WARMUP_COLDEST 1
int _Py_Quicken(PyCodeObject *code);
void _PyCode_Quicken(PyCodeObject *code);
/* Returns 1 if quickening occurs.
* -1 if an error occurs
* 0 otherwise */
static inline int
_Py_IncrementCountAndMaybeQuicken(PyCodeObject *code)
static inline void
_PyCode_Warmup(PyCodeObject *code)
{
if (code->co_warmup != 0) {
code->co_warmup++;
if (code->co_warmup == 0) {
return _Py_Quicken(code) ? -1 : 1;
_PyCode_Quicken(code);
}
}
return 0;
}
extern Py_ssize_t _Py_QuickenedCount;
@ -225,6 +217,7 @@ PyAPI_FUNC(PyCodeObject *) _PyCode_New(struct _PyCodeConstructor *);
extern PyObject* _PyCode_GetVarnames(PyCodeObject *);
extern PyObject* _PyCode_GetCellvars(PyCodeObject *);
extern PyObject* _PyCode_GetFreevars(PyCodeObject *);
extern PyObject* _PyCode_GetCode(PyCodeObject *);
/* Return the ending source code line number from a bytecode index. */
extern int _PyCode_Addr2EndLine(PyCodeObject *, int);

286
Include/opcode.h generated
View File

@ -115,38 +115,38 @@ extern "C" {
#define CALL 171
#define KW_NAMES 172
#define BINARY_OP_ADAPTIVE 3
#define BINARY_OP_ADD_INT 4
#define BINARY_OP_ADD_FLOAT 5
#define BINARY_OP_ADD_FLOAT 4
#define BINARY_OP_ADD_INT 5
#define BINARY_OP_ADD_UNICODE 6
#define BINARY_OP_INPLACE_ADD_UNICODE 7
#define BINARY_OP_MULTIPLY_INT 8
#define BINARY_OP_MULTIPLY_FLOAT 13
#define BINARY_OP_SUBTRACT_INT 14
#define BINARY_OP_SUBTRACT_FLOAT 16
#define COMPARE_OP_ADAPTIVE 17
#define COMPARE_OP_FLOAT_JUMP 18
#define COMPARE_OP_INT_JUMP 19
#define COMPARE_OP_STR_JUMP 20
#define BINARY_SUBSCR_ADAPTIVE 21
#define BINARY_SUBSCR_GETITEM 22
#define BINARY_SUBSCR_LIST_INT 23
#define BINARY_SUBSCR_TUPLE_INT 24
#define BINARY_SUBSCR_DICT 26
#define STORE_SUBSCR_ADAPTIVE 27
#define STORE_SUBSCR_LIST_INT 28
#define STORE_SUBSCR_DICT 29
#define CALL_ADAPTIVE 34
#define CALL_PY_EXACT_ARGS 36
#define CALL_PY_WITH_DEFAULTS 37
#define JUMP_ABSOLUTE_QUICK 38
#define LOAD_ATTR_ADAPTIVE 39
#define LOAD_ATTR_INSTANCE_VALUE 40
#define LOAD_ATTR_WITH_HINT 41
#define LOAD_ATTR_SLOT 42
#define LOAD_ATTR_MODULE 43
#define BINARY_OP_MULTIPLY_FLOAT 8
#define BINARY_OP_MULTIPLY_INT 13
#define BINARY_OP_SUBTRACT_FLOAT 14
#define BINARY_OP_SUBTRACT_INT 16
#define BINARY_SUBSCR_ADAPTIVE 17
#define BINARY_SUBSCR_DICT 18
#define BINARY_SUBSCR_GETITEM 19
#define BINARY_SUBSCR_LIST_INT 20
#define BINARY_SUBSCR_TUPLE_INT 21
#define CALL_ADAPTIVE 22
#define CALL_PY_EXACT_ARGS 23
#define CALL_PY_WITH_DEFAULTS 24
#define COMPARE_OP_ADAPTIVE 26
#define COMPARE_OP_FLOAT_JUMP 27
#define COMPARE_OP_INT_JUMP 28
#define COMPARE_OP_STR_JUMP 29
#define JUMP_ABSOLUTE_QUICK 34
#define LOAD_ATTR_ADAPTIVE 36
#define LOAD_ATTR_INSTANCE_VALUE 37
#define LOAD_ATTR_MODULE 38
#define LOAD_ATTR_SLOT 39
#define LOAD_ATTR_WITH_HINT 40
#define LOAD_CONST__LOAD_FAST 41
#define LOAD_FAST__LOAD_CONST 42
#define LOAD_FAST__LOAD_FAST 43
#define LOAD_GLOBAL_ADAPTIVE 44
#define LOAD_GLOBAL_MODULE 45
#define LOAD_GLOBAL_BUILTIN 46
#define LOAD_GLOBAL_BUILTIN 45
#define LOAD_GLOBAL_MODULE 46
#define LOAD_METHOD_ADAPTIVE 47
#define LOAD_METHOD_CLASS 48
#define LOAD_METHOD_MODULE 55
@ -154,38 +154,40 @@ extern "C" {
#define LOAD_METHOD_WITH_DICT 57
#define LOAD_METHOD_WITH_VALUES 58
#define PRECALL_ADAPTIVE 59
#define PRECALL_BUILTIN_CLASS 62
#define PRECALL_NO_KW_BUILTIN_O 63
#define PRECALL_NO_KW_BUILTIN_FAST 64
#define PRECALL_BUILTIN_FAST_WITH_KEYWORDS 65
#define PRECALL_NO_KW_LEN 66
#define PRECALL_BOUND_METHOD 62
#define PRECALL_BUILTIN_CLASS 63
#define PRECALL_BUILTIN_FAST_WITH_KEYWORDS 64
#define PRECALL_NO_KW_BUILTIN_FAST 65
#define PRECALL_NO_KW_BUILTIN_O 66
#define PRECALL_NO_KW_ISINSTANCE 67
#define PRECALL_NO_KW_LIST_APPEND 72
#define PRECALL_NO_KW_METHOD_DESCRIPTOR_O 73
#define PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 76
#define PRECALL_NO_KW_STR_1 77
#define PRECALL_NO_KW_TUPLE_1 78
#define PRECALL_NO_KW_TYPE_1 79
#define PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST 80
#define PRECALL_BOUND_METHOD 81
#define PRECALL_NO_KW_LEN 72
#define PRECALL_NO_KW_LIST_APPEND 73
#define PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST 76
#define PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 77
#define PRECALL_NO_KW_METHOD_DESCRIPTOR_O 78
#define PRECALL_NO_KW_STR_1 79
#define PRECALL_NO_KW_TUPLE_1 80
#define PRECALL_NO_KW_TYPE_1 81
#define PRECALL_PYFUNC 140
#define RESUME_QUICK 141
#define STORE_ATTR_ADAPTIVE 143
#define STORE_ATTR_INSTANCE_VALUE 150
#define STORE_ATTR_SLOT 153
#define STORE_ATTR_WITH_HINT 154
#define UNPACK_SEQUENCE_ADAPTIVE 158
#define UNPACK_SEQUENCE_LIST 159
#define UNPACK_SEQUENCE_TUPLE 161
#define UNPACK_SEQUENCE_TWO_TUPLE 167
#define LOAD_FAST__LOAD_FAST 168
#define STORE_FAST__LOAD_FAST 169
#define LOAD_FAST__LOAD_CONST 170
#define LOAD_CONST__LOAD_FAST 173
#define STORE_FAST__STORE_FAST 174
#define STORE_FAST__LOAD_FAST 158
#define STORE_FAST__STORE_FAST 159
#define STORE_SUBSCR_ADAPTIVE 161
#define STORE_SUBSCR_DICT 167
#define STORE_SUBSCR_LIST_INT 168
#define UNPACK_SEQUENCE_ADAPTIVE 169
#define UNPACK_SEQUENCE_LIST 170
#define UNPACK_SEQUENCE_TUPLE 173
#define UNPACK_SEQUENCE_TWO_TUPLE 174
#define DO_TRACING 255
extern const uint8_t _PyOpcode_InlineCacheEntries[256];
extern const uint8_t _PyOpcode_Caches[256];
extern const uint8_t _PyOpcode_Deopt[256];
#ifdef NEED_OPCODE_TABLES
static const uint32_t _PyOpcode_RelativeJump[8] = {
@ -209,7 +211,7 @@ static const uint32_t _PyOpcode_Jump[8] = {
0U,
};
const uint8_t _PyOpcode_InlineCacheEntries[256] = {
const uint8_t _PyOpcode_Caches[256] = {
[BINARY_SUBSCR] = 4,
[STORE_SUBSCR] = 1,
[UNPACK_SEQUENCE] = 1,
@ -222,6 +224,184 @@ const uint8_t _PyOpcode_InlineCacheEntries[256] = {
[PRECALL] = 1,
[CALL] = 4,
};
const uint8_t _PyOpcode_Deopt[256] = {
[ASYNC_GEN_WRAP] = ASYNC_GEN_WRAP,
[BEFORE_ASYNC_WITH] = BEFORE_ASYNC_WITH,
[BEFORE_WITH] = BEFORE_WITH,
[BINARY_OP] = BINARY_OP,
[BINARY_OP_ADAPTIVE] = BINARY_OP,
[BINARY_OP_ADD_FLOAT] = BINARY_OP,
[BINARY_OP_ADD_INT] = BINARY_OP,
[BINARY_OP_ADD_UNICODE] = BINARY_OP,
[BINARY_OP_INPLACE_ADD_UNICODE] = BINARY_OP,
[BINARY_OP_MULTIPLY_FLOAT] = BINARY_OP,
[BINARY_OP_MULTIPLY_INT] = BINARY_OP,
[BINARY_OP_SUBTRACT_FLOAT] = BINARY_OP,
[BINARY_OP_SUBTRACT_INT] = BINARY_OP,
[BINARY_SUBSCR] = BINARY_SUBSCR,
[BINARY_SUBSCR_ADAPTIVE] = BINARY_SUBSCR,
[BINARY_SUBSCR_DICT] = BINARY_SUBSCR,
[BINARY_SUBSCR_GETITEM] = BINARY_SUBSCR,
[BINARY_SUBSCR_LIST_INT] = BINARY_SUBSCR,
[BINARY_SUBSCR_TUPLE_INT] = BINARY_SUBSCR,
[BUILD_CONST_KEY_MAP] = BUILD_CONST_KEY_MAP,
[BUILD_LIST] = BUILD_LIST,
[BUILD_MAP] = BUILD_MAP,
[BUILD_SET] = BUILD_SET,
[BUILD_SLICE] = BUILD_SLICE,
[BUILD_STRING] = BUILD_STRING,
[BUILD_TUPLE] = BUILD_TUPLE,
[CACHE] = CACHE,
[CALL] = CALL,
[CALL_ADAPTIVE] = CALL,
[CALL_FUNCTION_EX] = CALL_FUNCTION_EX,
[CALL_PY_EXACT_ARGS] = CALL,
[CALL_PY_WITH_DEFAULTS] = CALL,
[COMPARE_OP] = COMPARE_OP,
[COMPARE_OP_ADAPTIVE] = COMPARE_OP,
[COMPARE_OP_FLOAT_JUMP] = COMPARE_OP,
[COMPARE_OP_INT_JUMP] = COMPARE_OP,
[COMPARE_OP_STR_JUMP] = COMPARE_OP,
[CONTAINS_OP] = CONTAINS_OP,
[COPY] = COPY,
[COPY_FREE_VARS] = COPY_FREE_VARS,
[DELETE_ATTR] = DELETE_ATTR,
[DELETE_DEREF] = DELETE_DEREF,
[DELETE_FAST] = DELETE_FAST,
[DELETE_GLOBAL] = DELETE_GLOBAL,
[DELETE_NAME] = DELETE_NAME,
[DELETE_SUBSCR] = DELETE_SUBSCR,
[DICT_MERGE] = DICT_MERGE,
[DICT_UPDATE] = DICT_UPDATE,
[END_ASYNC_FOR] = END_ASYNC_FOR,
[EXTENDED_ARG] = EXTENDED_ARG,
[FORMAT_VALUE] = FORMAT_VALUE,
[FOR_ITER] = FOR_ITER,
[GET_AITER] = GET_AITER,
[GET_ANEXT] = GET_ANEXT,
[GET_AWAITABLE] = GET_AWAITABLE,
[GET_ITER] = GET_ITER,
[GET_LEN] = GET_LEN,
[GET_YIELD_FROM_ITER] = GET_YIELD_FROM_ITER,
[IMPORT_FROM] = IMPORT_FROM,
[IMPORT_NAME] = IMPORT_NAME,
[IMPORT_STAR] = IMPORT_STAR,
[IS_OP] = IS_OP,
[JUMP_ABSOLUTE] = JUMP_ABSOLUTE,
[JUMP_ABSOLUTE_QUICK] = JUMP_ABSOLUTE,
[JUMP_FORWARD] = JUMP_FORWARD,
[JUMP_IF_FALSE_OR_POP] = JUMP_IF_FALSE_OR_POP,
[JUMP_IF_NOT_EG_MATCH] = JUMP_IF_NOT_EG_MATCH,
[JUMP_IF_NOT_EXC_MATCH] = JUMP_IF_NOT_EXC_MATCH,
[JUMP_IF_TRUE_OR_POP] = JUMP_IF_TRUE_OR_POP,
[JUMP_NO_INTERRUPT] = JUMP_NO_INTERRUPT,
[KW_NAMES] = KW_NAMES,
[LIST_APPEND] = LIST_APPEND,
[LIST_EXTEND] = LIST_EXTEND,
[LIST_TO_TUPLE] = LIST_TO_TUPLE,
[LOAD_ASSERTION_ERROR] = LOAD_ASSERTION_ERROR,
[LOAD_ATTR] = LOAD_ATTR,
[LOAD_ATTR_ADAPTIVE] = LOAD_ATTR,
[LOAD_ATTR_INSTANCE_VALUE] = LOAD_ATTR,
[LOAD_ATTR_MODULE] = LOAD_ATTR,
[LOAD_ATTR_SLOT] = LOAD_ATTR,
[LOAD_ATTR_WITH_HINT] = LOAD_ATTR,
[LOAD_BUILD_CLASS] = LOAD_BUILD_CLASS,
[LOAD_CLASSDEREF] = LOAD_CLASSDEREF,
[LOAD_CLOSURE] = LOAD_CLOSURE,
[LOAD_CONST] = LOAD_CONST,
[LOAD_CONST__LOAD_FAST] = LOAD_CONST,
[LOAD_DEREF] = LOAD_DEREF,
[LOAD_FAST] = LOAD_FAST,
[LOAD_FAST__LOAD_CONST] = LOAD_FAST,
[LOAD_FAST__LOAD_FAST] = LOAD_FAST,
[LOAD_GLOBAL] = LOAD_GLOBAL,
[LOAD_GLOBAL_ADAPTIVE] = LOAD_GLOBAL,
[LOAD_GLOBAL_BUILTIN] = LOAD_GLOBAL,
[LOAD_GLOBAL_MODULE] = LOAD_GLOBAL,
[LOAD_METHOD] = LOAD_METHOD,
[LOAD_METHOD_ADAPTIVE] = LOAD_METHOD,
[LOAD_METHOD_CLASS] = LOAD_METHOD,
[LOAD_METHOD_MODULE] = LOAD_METHOD,
[LOAD_METHOD_NO_DICT] = LOAD_METHOD,
[LOAD_METHOD_WITH_DICT] = LOAD_METHOD,
[LOAD_METHOD_WITH_VALUES] = LOAD_METHOD,
[LOAD_NAME] = LOAD_NAME,
[MAKE_CELL] = MAKE_CELL,
[MAKE_FUNCTION] = MAKE_FUNCTION,
[MAP_ADD] = MAP_ADD,
[MATCH_CLASS] = MATCH_CLASS,
[MATCH_KEYS] = MATCH_KEYS,
[MATCH_MAPPING] = MATCH_MAPPING,
[MATCH_SEQUENCE] = MATCH_SEQUENCE,
[NOP] = NOP,
[POP_EXCEPT] = POP_EXCEPT,
[POP_JUMP_IF_FALSE] = POP_JUMP_IF_FALSE,
[POP_JUMP_IF_NONE] = POP_JUMP_IF_NONE,
[POP_JUMP_IF_NOT_NONE] = POP_JUMP_IF_NOT_NONE,
[POP_JUMP_IF_TRUE] = POP_JUMP_IF_TRUE,
[POP_TOP] = POP_TOP,
[PRECALL] = PRECALL,
[PRECALL_ADAPTIVE] = PRECALL,
[PRECALL_BOUND_METHOD] = PRECALL,
[PRECALL_BUILTIN_CLASS] = PRECALL,
[PRECALL_BUILTIN_FAST_WITH_KEYWORDS] = PRECALL,
[PRECALL_NO_KW_BUILTIN_FAST] = PRECALL,
[PRECALL_NO_KW_BUILTIN_O] = PRECALL,
[PRECALL_NO_KW_ISINSTANCE] = PRECALL,
[PRECALL_NO_KW_LEN] = PRECALL,
[PRECALL_NO_KW_LIST_APPEND] = PRECALL,
[PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST] = PRECALL,
[PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = PRECALL,
[PRECALL_NO_KW_METHOD_DESCRIPTOR_O] = PRECALL,
[PRECALL_NO_KW_STR_1] = PRECALL,
[PRECALL_NO_KW_TUPLE_1] = PRECALL,
[PRECALL_NO_KW_TYPE_1] = PRECALL,
[PRECALL_PYFUNC] = PRECALL,
[PREP_RERAISE_STAR] = PREP_RERAISE_STAR,
[PRINT_EXPR] = PRINT_EXPR,
[PUSH_EXC_INFO] = PUSH_EXC_INFO,
[PUSH_NULL] = PUSH_NULL,
[RAISE_VARARGS] = RAISE_VARARGS,
[RERAISE] = RERAISE,
[RESUME] = RESUME,
[RESUME_QUICK] = RESUME,
[RETURN_GENERATOR] = RETURN_GENERATOR,
[RETURN_VALUE] = RETURN_VALUE,
[SEND] = SEND,
[SETUP_ANNOTATIONS] = SETUP_ANNOTATIONS,
[SET_ADD] = SET_ADD,
[SET_UPDATE] = SET_UPDATE,
[STORE_ATTR] = STORE_ATTR,
[STORE_ATTR_ADAPTIVE] = STORE_ATTR,
[STORE_ATTR_INSTANCE_VALUE] = STORE_ATTR,
[STORE_ATTR_SLOT] = STORE_ATTR,
[STORE_ATTR_WITH_HINT] = STORE_ATTR,
[STORE_DEREF] = STORE_DEREF,
[STORE_FAST] = STORE_FAST,
[STORE_FAST__LOAD_FAST] = STORE_FAST,
[STORE_FAST__STORE_FAST] = STORE_FAST,
[STORE_GLOBAL] = STORE_GLOBAL,
[STORE_NAME] = STORE_NAME,
[STORE_SUBSCR] = STORE_SUBSCR,
[STORE_SUBSCR_ADAPTIVE] = STORE_SUBSCR,
[STORE_SUBSCR_DICT] = STORE_SUBSCR,
[STORE_SUBSCR_LIST_INT] = STORE_SUBSCR,
[SWAP] = SWAP,
[UNARY_INVERT] = UNARY_INVERT,
[UNARY_NEGATIVE] = UNARY_NEGATIVE,
[UNARY_NOT] = UNARY_NOT,
[UNARY_POSITIVE] = UNARY_POSITIVE,
[UNPACK_EX] = UNPACK_EX,
[UNPACK_SEQUENCE] = UNPACK_SEQUENCE,
[UNPACK_SEQUENCE_ADAPTIVE] = UNPACK_SEQUENCE,
[UNPACK_SEQUENCE_LIST] = UNPACK_SEQUENCE,
[UNPACK_SEQUENCE_TUPLE] = UNPACK_SEQUENCE,
[UNPACK_SEQUENCE_TWO_TUPLE] = UNPACK_SEQUENCE,
[WITH_EXCEPT_START] = WITH_EXCEPT_START,
[YIELD_VALUE] = YIELD_VALUE,
};
#endif /* OPCODE_TABLES */
#define HAS_CONST(op) (false\

94
Lib/opcode.py
View File

@ -229,77 +229,111 @@ _nb_ops = [
("NB_INPLACE_XOR", "^="),
]
_specialized_instructions = [
_specializations = {
"BINARY_OP": [
"BINARY_OP_ADAPTIVE",
"BINARY_OP_ADD_INT",
"BINARY_OP_ADD_FLOAT",
"BINARY_OP_ADD_INT",
"BINARY_OP_ADD_UNICODE",
"BINARY_OP_INPLACE_ADD_UNICODE",
"BINARY_OP_MULTIPLY_INT",
"BINARY_OP_MULTIPLY_FLOAT",
"BINARY_OP_SUBTRACT_INT",
"BINARY_OP_MULTIPLY_INT",
"BINARY_OP_SUBTRACT_FLOAT",
"BINARY_OP_SUBTRACT_INT",
],
"BINARY_SUBSCR": [
"BINARY_SUBSCR_ADAPTIVE",
"BINARY_SUBSCR_DICT",
"BINARY_SUBSCR_GETITEM",
"BINARY_SUBSCR_LIST_INT",
"BINARY_SUBSCR_TUPLE_INT",
],
"CALL": [
"CALL_ADAPTIVE",
"CALL_PY_EXACT_ARGS",
"CALL_PY_WITH_DEFAULTS",
],
"COMPARE_OP": [
"COMPARE_OP_ADAPTIVE",
"COMPARE_OP_FLOAT_JUMP",
"COMPARE_OP_INT_JUMP",
"COMPARE_OP_STR_JUMP",
"BINARY_SUBSCR_ADAPTIVE",
"BINARY_SUBSCR_GETITEM",
"BINARY_SUBSCR_LIST_INT",
"BINARY_SUBSCR_TUPLE_INT",
"BINARY_SUBSCR_DICT",
"STORE_SUBSCR_ADAPTIVE",
"STORE_SUBSCR_LIST_INT",
"STORE_SUBSCR_DICT",
"CALL_ADAPTIVE",
"CALL_PY_EXACT_ARGS",
"CALL_PY_WITH_DEFAULTS",
],
"JUMP_ABSOLUTE": [
"JUMP_ABSOLUTE_QUICK",
],
"LOAD_ATTR": [
"LOAD_ATTR_ADAPTIVE",
"LOAD_ATTR_INSTANCE_VALUE",
"LOAD_ATTR_WITH_HINT",
"LOAD_ATTR_SLOT",
"LOAD_ATTR_MODULE",
"LOAD_ATTR_SLOT",
"LOAD_ATTR_WITH_HINT",
],
"LOAD_CONST": [
"LOAD_CONST__LOAD_FAST",
],
"LOAD_FAST": [
"LOAD_FAST__LOAD_CONST",
"LOAD_FAST__LOAD_FAST",
],
"LOAD_GLOBAL": [
"LOAD_GLOBAL_ADAPTIVE",
"LOAD_GLOBAL_MODULE",
"LOAD_GLOBAL_BUILTIN",
"LOAD_GLOBAL_MODULE",
],
"LOAD_METHOD": [
"LOAD_METHOD_ADAPTIVE",
"LOAD_METHOD_CLASS",
"LOAD_METHOD_MODULE",
"LOAD_METHOD_NO_DICT",
"LOAD_METHOD_WITH_DICT",
"LOAD_METHOD_WITH_VALUES",
],
"PRECALL": [
"PRECALL_ADAPTIVE",
"PRECALL_BOUND_METHOD",
"PRECALL_BUILTIN_CLASS",
"PRECALL_NO_KW_BUILTIN_O",
"PRECALL_NO_KW_BUILTIN_FAST",
"PRECALL_BUILTIN_FAST_WITH_KEYWORDS",
"PRECALL_NO_KW_LEN",
"PRECALL_NO_KW_BUILTIN_FAST",
"PRECALL_NO_KW_BUILTIN_O",
"PRECALL_NO_KW_ISINSTANCE",
"PRECALL_NO_KW_LEN",
"PRECALL_NO_KW_LIST_APPEND",
"PRECALL_NO_KW_METHOD_DESCRIPTOR_O",
"PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST",
"PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS",
"PRECALL_NO_KW_METHOD_DESCRIPTOR_O",
"PRECALL_NO_KW_STR_1",
"PRECALL_NO_KW_TUPLE_1",
"PRECALL_NO_KW_TYPE_1",
"PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST",
"PRECALL_BOUND_METHOD",
"PRECALL_PYFUNC",
],
"RESUME": [
"RESUME_QUICK",
],
"STORE_ATTR": [
"STORE_ATTR_ADAPTIVE",
"STORE_ATTR_INSTANCE_VALUE",
"STORE_ATTR_SLOT",
"STORE_ATTR_WITH_HINT",
],
"STORE_FAST": [
"STORE_FAST__LOAD_FAST",
"STORE_FAST__STORE_FAST",
],
"STORE_SUBSCR": [
"STORE_SUBSCR_ADAPTIVE",
"STORE_SUBSCR_DICT",
"STORE_SUBSCR_LIST_INT",
],
"UNPACK_SEQUENCE": [
"UNPACK_SEQUENCE_ADAPTIVE",
"UNPACK_SEQUENCE_LIST",
"UNPACK_SEQUENCE_TUPLE",
"UNPACK_SEQUENCE_TWO_TUPLE",
# Super instructions
"LOAD_FAST__LOAD_FAST",
"STORE_FAST__LOAD_FAST",
"LOAD_FAST__LOAD_CONST",
"LOAD_CONST__LOAD_FAST",
"STORE_FAST__STORE_FAST",
],
}
_specialized_instructions = [
opcode for family in _specializations.values() for opcode in family
]
_specialization_stats = [
"success",

3
Lib/test/test_compile.py
View File

@ -641,7 +641,7 @@ if 1:
self.check_constant(f1, frozenset({0}))
self.assertTrue(f1(0))
# Merging equal co_linetable and co_code is not a strict requirement
# Merging equal co_linetable is not a strict requirement
# for the Python semantics, it's a more an implementation detail.
@support.cpython_only
def test_merge_code_attrs(self):
@ -650,7 +650,6 @@ if 1:
f2 = lambda a: a.b.c
self.assertIs(f1.__code__.co_linetable, f2.__code__.co_linetable)
self.assertIs(f1.__code__.co_code, f2.__code__.co_code)
# Stripping unused constants is not a strict requirement for the
# Python semantics, it's a more an implementation detail.

2
Misc/NEWS.d/next/Core and Builtins/2022-03-16-11-05-35.bpo-46841.yUoIHg.rst Normal file
View File

@ -0,0 +1,2 @@
Quicken bytecode in-place by storing it as part of the corresponding
``PyCodeObject``.

10
Objects/clinic/codeobject.c.h generated
View File

@ -203,12 +203,12 @@ code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje
int co_stacksize = self->co_stacksize;
int co_flags = self->co_flags;
int co_firstlineno = self->co_firstlineno;
PyBytesObject *co_code = (PyBytesObject *)self->co_code;
PyBytesObject *co_code = NULL;
PyObject *co_consts = self->co_consts;
PyObject *co_names = self->co_names;
PyObject *co_varnames = self->co_varnames;
PyObject *co_freevars = self->co_freevars;
PyObject *co_cellvars = self->co_cellvars;
PyObject *co_varnames = NULL;
PyObject *co_freevars = NULL;
PyObject *co_cellvars = NULL;
PyObject *co_filename = self->co_filename;
PyObject *co_name = self->co_name;
PyObject *co_qualname = self->co_qualname;
@ -456,4 +456,4 @@ code__varname_from_oparg(PyCodeObject *self, PyObject *const *args, Py_ssize_t n
exit:
return return_value;
}
/*[clinic end generated code: output=9e8c4a19474ec520 input=a9049054013a1b77]*/
/*[clinic end generated code: output=b1b83a70ffc5b7cd input=a9049054013a1b77]*/

171
Objects/codeobject.c
View File

@ -305,9 +305,6 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con)
co->co_qualname = con->qualname;
co->co_flags = con->flags;
Py_INCREF(con->code);
co->co_code = con->code;
co->co_firstinstr = (_Py_CODEUNIT *)PyBytes_AS_STRING(con->code);
co->co_firstlineno = con->firstlineno;
Py_INCREF(con->linetable);
co->co_linetable = con->linetable;
@ -341,16 +338,14 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con)
co->co_nplaincellvars = nplaincellvars;
co->co_ncellvars = ncellvars;
co->co_nfreevars = nfreevars;
co->co_varnames = NULL;
co->co_cellvars = NULL;
co->co_freevars = NULL;
/* not set */
co->co_weakreflist = NULL;
co->co_extra = NULL;
co->co_warmup = QUICKENING_INITIAL_WARMUP_VALUE;
co->co_quickened = NULL;
memcpy(_PyCode_CODE(co), PyBytes_AS_STRING(con->code),
PyBytes_GET_SIZE(con->code));
}
/* The caller is responsible for ensuring that the given data is valid. */
@ -386,7 +381,8 @@ _PyCode_New(struct _PyCodeConstructor *con)
con->columntable = Py_None;
}
PyCodeObject *co = PyObject_New(PyCodeObject, &PyCode_Type);
Py_ssize_t size = PyBytes_GET_SIZE(con->code) / sizeof(_Py_CODEUNIT);
PyCodeObject *co = PyObject_NewVar(PyCodeObject, &PyCode_Type, size);
if (co == NULL) {
PyErr_NoMemory();
return NULL;
@ -521,13 +517,6 @@ PyCode_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount,
goto error;
}
Py_INCREF(varnames);
co->co_varnames = varnames;
Py_INCREF(cellvars);
co->co_cellvars = cellvars;
Py_INCREF(freevars);
co->co_freevars = freevars;
error:
Py_XDECREF(localsplusnames);
Py_XDECREF(localspluskinds);
@ -611,7 +600,7 @@ PyCode_Addr2Line(PyCodeObject *co, int addrq)
if (addrq < 0) {
return co->co_firstlineno;
}
assert(addrq >= 0 && addrq < PyBytes_GET_SIZE(co->co_code));
assert(addrq >= 0 && addrq < _PyCode_NBYTES(co));
PyCodeAddressRange bounds;
_PyCode_InitAddressRange(co, &bounds);
return _PyCode_CheckLineNumber(addrq, &bounds);
@ -639,7 +628,7 @@ _PyCode_Addr2EndLine(PyCodeObject* co, int addrq)
return -1;
}
assert(addrq >= 0 && addrq < PyBytes_GET_SIZE(co->co_code));
assert(addrq >= 0 && addrq < _PyCode_NBYTES(co));
PyCodeAddressRange bounds;
_PyCode_InitEndAddressRange(co, &bounds);
return _PyCode_CheckLineNumber(addrq, &bounds);
@ -995,7 +984,7 @@ _source_offset_converter(int* value) {
static PyObject*
positionsiter_next(positionsiterator* pi)
{
if (pi->pi_offset >= PyBytes_GET_SIZE(pi->pi_code->co_code)) {
if (pi->pi_offset >= _PyCode_NBYTES(pi->pi_code)) {
return NULL;
}
@ -1151,46 +1140,39 @@ _PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra)
PyObject *
_PyCode_GetVarnames(PyCodeObject *co)
{
if (co->co_varnames == NULL) {
// PyCodeObject owns this reference.
co->co_varnames = get_localsplus_names(co, CO_FAST_LOCAL,
co->co_nlocals);
if (co->co_varnames == NULL) {
return NULL;
}
}
Py_INCREF(co->co_varnames);
return co->co_varnames;
return get_localsplus_names(co, CO_FAST_LOCAL, co->co_nlocals);
}
PyObject *
_PyCode_GetCellvars(PyCodeObject *co)
{
if (co->co_cellvars == NULL) {
// PyCodeObject owns this reference.
co->co_cellvars = get_localsplus_names(co, CO_FAST_CELL,
co->co_ncellvars);
if (co->co_cellvars == NULL) {
return NULL;
}
}
Py_INCREF(co->co_cellvars);
return co->co_cellvars;
return get_localsplus_names(co, CO_FAST_CELL, co->co_ncellvars);
}
PyObject *
_PyCode_GetFreevars(PyCodeObject *co)
{
if (co->co_freevars == NULL) {
// PyCodeObject owns this reference.
co->co_freevars = get_localsplus_names(co, CO_FAST_FREE,
co->co_nfreevars);
if (co->co_freevars == NULL) {
return get_localsplus_names(co, CO_FAST_FREE, co->co_nfreevars);
}
PyObject *
_PyCode_GetCode(PyCodeObject *co)
{
PyObject *code = PyBytes_FromStringAndSize(NULL, _PyCode_NBYTES(co));
if (code == NULL) {
return NULL;
}
_Py_CODEUNIT *instructions = (_Py_CODEUNIT *)PyBytes_AS_STRING(code);
for (int i = 0; i < Py_SIZE(co); i++) {
_Py_CODEUNIT instruction = _PyCode_CODE(co)[i];
int opcode = _PyOpcode_Deopt[_Py_OPCODE(instruction)];
int caches = _PyOpcode_Caches[opcode];
instructions[i] = _Py_MAKECODEUNIT(opcode, _Py_OPARG(instruction));
while (caches--) {
instructions[++i] = _Py_MAKECODEUNIT(CACHE, 0);
}
Py_INCREF(co->co_freevars);
return co->co_freevars;
}
return code;
}
@ -1348,14 +1330,10 @@ code_dealloc(PyCodeObject *co)
PyMem_Free(co_extra);
}
Py_XDECREF(co->co_code);
Py_XDECREF(co->co_consts);
Py_XDECREF(co->co_names);
Py_XDECREF(co->co_localsplusnames);
Py_XDECREF(co->co_localspluskinds);
Py_XDECREF(co->co_varnames);
Py_XDECREF(co->co_freevars);
Py_XDECREF(co->co_cellvars);
Py_XDECREF(co->co_filename);
Py_XDECREF(co->co_name);
Py_XDECREF(co->co_qualname);
@ -1363,10 +1341,10 @@ code_dealloc(PyCodeObject *co)
Py_XDECREF(co->co_endlinetable);
Py_XDECREF(co->co_columntable);
Py_XDECREF(co->co_exceptiontable);
if (co->co_weakreflist != NULL)
if (co->co_weakreflist != NULL) {
PyObject_ClearWeakRefs((PyObject*)co);
if (co->co_quickened) {
PyMem_Free(co->co_quickened);
}
if (co->co_warmup == 0) {
_Py_QuickenedCount--;
}
PyObject_Free(co);
@ -1420,8 +1398,21 @@ code_richcompare(PyObject *self, PyObject *other, int op)
if (!eq) goto unequal;
eq = co->co_firstlineno == cp->co_firstlineno;
if (!eq) goto unequal;
eq = PyObject_RichCompareBool(co->co_code, cp->co_code, Py_EQ);
if (eq <= 0) goto unequal;
PyObject *co_code = _PyCode_GetCode(co);
if (co_code == NULL) {
return NULL;
}
PyObject *cp_code = _PyCode_GetCode(cp);
if (cp_code == NULL) {
Py_DECREF(co_code);
return NULL;
}
eq = PyObject_RichCompareBool(co_code, cp_code, Py_EQ);
Py_DECREF(co_code);
Py_DECREF(cp_code);
if (eq <= 0) {
goto unequal;
}
/* compare constants */
consts1 = _PyCode_ConstantKey(co->co_consts);
@ -1465,18 +1456,16 @@ code_richcompare(PyObject *self, PyObject *other, int op)
static Py_hash_t
code_hash(PyCodeObject *co)
{
Py_hash_t h, h0, h1, h2, h3, h4;
Py_hash_t h, h0, h1, h2, h3;
h0 = PyObject_Hash(co->co_name);
if (h0 == -1) return -1;
h1 = PyObject_Hash(co->co_code);
h1 = PyObject_Hash(co->co_consts);
if (h1 == -1) return -1;
h2 = PyObject_Hash(co->co_consts);
h2 = PyObject_Hash(co->co_names);
if (h2 == -1) return -1;
h3 = PyObject_Hash(co->co_names);
h3 = PyObject_Hash(co->co_localsplusnames);
if (h3 == -1) return -1;
h4 = PyObject_Hash(co->co_localsplusnames);
if (h4 == -1) return -1;
h = h0 ^ h1 ^ h2 ^ h3 ^ h4 ^
h = h0 ^ h1 ^ h2 ^ h3 ^
co->co_argcount ^ co->co_posonlyargcount ^ co->co_kwonlyargcount ^
co->co_flags;
if (h == -1) h = -2;
@ -1492,7 +1481,7 @@ static PyMemberDef code_memberlist[] = {
{"co_kwonlyargcount", T_INT, OFF(co_kwonlyargcount), READONLY},
{"co_stacksize", T_INT, OFF(co_stacksize), READONLY},
{"co_flags", T_INT, OFF(co_flags), READONLY},
{"co_code", T_OBJECT, OFF(co_code), READONLY},
{"co_nlocals", T_INT, OFF(co_nlocals), READONLY},
{"co_consts", T_OBJECT, OFF(co_consts), READONLY},
{"co_names", T_OBJECT, OFF(co_names), READONLY},
{"co_filename", T_OBJECT, OFF(co_filename), READONLY},
@ -1513,12 +1502,6 @@ code_getlnotab(PyCodeObject *code, void *closure)
return decode_linetable(code);
}
static PyObject *
code_getnlocals(PyCodeObject *code, void *closure)
{
return PyLong_FromLong(code->co_nlocals);
}
static PyObject *
code_getvarnames(PyCodeObject *code, void *closure)
{
@ -1538,23 +1521,26 @@ code_getfreevars(PyCodeObject *code, void *closure)
}
static PyObject *
code_getquickened(PyCodeObject *code, void *closure)
code_getcodeadaptive(PyCodeObject *code, void *closure)
{
if (code->co_quickened == NULL) {
Py_RETURN_NONE;
return PyMemoryView_FromMemory(code->co_code_adaptive, _PyCode_NBYTES(code),
PyBUF_READ);
}
return PyBytes_FromStringAndSize((char *)code->co_firstinstr,
PyBytes_Size(code->co_code));
static PyObject *
code_getcode(PyCodeObject *code, void *closure)
{
return _PyCode_GetCode(code);
}
static PyGetSetDef code_getsetlist[] = {
{"co_lnotab", (getter)code_getlnotab, NULL, NULL},
{"_co_code_adaptive", (getter)code_getcodeadaptive, NULL, NULL},
// The following old names are kept for backward compatibility.
{"co_nlocals", (getter)code_getnlocals, NULL, NULL},
{"co_varnames", (getter)code_getvarnames, NULL, NULL},
{"co_cellvars", (getter)code_getcellvars, NULL, NULL},
{"co_freevars", (getter)code_getfreevars, NULL, NULL},
{"_co_quickened", (getter)code_getquickened, NULL, NULL},
{"co_code", (getter)code_getcode, NULL, NULL},
{0}
};
@ -1562,7 +1548,7 @@ static PyGetSetDef code_getsetlist[] = {
static PyObject *
code_sizeof(PyCodeObject *co, PyObject *Py_UNUSED(args))
{
Py_ssize_t res = _PyObject_SIZE(Py_TYPE(co));
Py_ssize_t res = _PyObject_VAR_SIZE(Py_TYPE(co), Py_SIZE(co));
_PyCodeObjectExtra *co_extra = (_PyCodeObjectExtra*) co->co_extra;
if (co_extra != NULL) {
@ -1570,10 +1556,6 @@ code_sizeof(PyCodeObject *co, PyObject *Py_UNUSED(args))
(co_extra->ce_size-1) * sizeof(co_extra->ce_extras[0]);
}
if (co->co_quickened != NULL) {
res += PyBytes_GET_SIZE(co->co_code);
}
return PyLong_FromSsize_t(res);
}
@ -1594,12 +1576,12 @@ code.replace
co_stacksize: int(c_default="self->co_stacksize") = -1
co_flags: int(c_default="self->co_flags") = -1
co_firstlineno: int(c_default="self->co_firstlineno") = -1
co_code: PyBytesObject(c_default="(PyBytesObject *)self->co_code") = None
co_code: PyBytesObject(c_default="NULL") = None
co_consts: object(subclass_of="&PyTuple_Type", c_default="self->co_consts") = None
co_names: object(subclass_of="&PyTuple_Type", c_default="self->co_names") = None
co_varnames: object(subclass_of="&PyTuple_Type", c_default="self->co_varnames") = None
co_freevars: object(subclass_of="&PyTuple_Type", c_default="self->co_freevars") = None
co_cellvars: object(subclass_of="&PyTuple_Type", c_default="self->co_cellvars") = None
co_varnames: object(subclass_of="&PyTuple_Type", c_default="NULL") = None
co_freevars: object(subclass_of="&PyTuple_Type", c_default="NULL") = None
co_cellvars: object(subclass_of="&PyTuple_Type", c_default="NULL") = None
co_filename: unicode(c_default="self->co_filename") = None
co_name: unicode(c_default="self->co_name") = None
co_qualname: unicode(c_default="self->co_qualname") = None
@ -1622,7 +1604,7 @@ code_replace_impl(PyCodeObject *self, int co_argcount,
PyObject *co_name, PyObject *co_qualname,
PyBytesObject *co_linetable, PyObject *co_endlinetable,
PyObject *co_columntable, PyBytesObject *co_exceptiontable)
/*[clinic end generated code: output=f046bf0be3bab91f input=a63d09f248f00794]*/
/*[clinic end generated code: output=f046bf0be3bab91f input=78dbe204dbd06c2f]*/
{
#define CHECK_INT_ARG(ARG) \
if (ARG < 0) { \
@ -1641,6 +1623,15 @@ code_replace_impl(PyCodeObject *self, int co_argcount,
#undef CHECK_INT_ARG
PyObject *code = NULL;
if (co_code == NULL) {
code = _PyCode_GetCode(self);
if (code == NULL) {
return NULL;
}
co_code = (PyBytesObject *)code;
}
if (PySys_Audit("code.__new__", "OOOiiiiii",
co_code, co_filename, co_name, co_argcount,
co_posonlyargcount, co_kwonlyargcount, co_nlocals,
@ -1694,6 +1685,7 @@ code_replace_impl(PyCodeObject *self, int co_argcount,
(PyObject*)co_exceptiontable);
error:
Py_XDECREF(code);
Py_XDECREF(varnames);
Py_XDECREF(cellvars);
Py_XDECREF(freevars);
@ -1737,8 +1729,8 @@ static struct PyMethodDef code_methods[] = {
PyTypeObject PyCode_Type = {
PyVarObject_HEAD_INIT(&PyType_Type, 0)
"code",
sizeof(PyCodeObject),
0,
offsetof(PyCodeObject, co_code_adaptive),
sizeof(_Py_CODEUNIT),
(destructor)code_dealloc, /* tp_dealloc */
0, /* tp_vectorcall_offset */
0, /* tp_getattr */
@ -1913,15 +1905,12 @@ _PyCode_ConstantKey(PyObject *op)
void
_PyStaticCode_Dealloc(PyCodeObject *co)
{
if (co->co_quickened) {
PyMem_Free(co->co_quickened);
co->co_quickened = NULL;
if (co->co_warmup == 0) {
_Py_QuickenedCount--;
}
co->co_warmup = QUICKENING_INITIAL_WARMUP_VALUE;
PyMem_Free(co->co_extra);
co->co_extra = NULL;
co->co_firstinstr = (_Py_CODEUNIT *)PyBytes_AS_STRING(co->co_code);
if (co->co_weakreflist != NULL) {
PyObject_ClearWeakRefs((PyObject *)co);
co->co_weakreflist = NULL;

39
Objects/frameobject.c
View File

@ -105,8 +105,9 @@ frame_getback(PyFrameObject *f, void *closure)
return res;
}
/* Given the index of the effective opcode,
scan back to construct the oparg with EXTENDED_ARG */
// Given the index of the effective opcode, scan back to construct the oparg
// with EXTENDED_ARG. This only works correctly with *unquickened* code,
// obtained via a call to _PyCode_GetCode!
static unsigned int
get_arg(const _Py_CODEUNIT *codestr, Py_ssize_t i)
{
@ -170,13 +171,17 @@ top_of_stack(int64_t stack)
static int64_t *
mark_stacks(PyCodeObject *code_obj, int len)
{
const _Py_CODEUNIT *code =
(const _Py_CODEUNIT *)PyBytes_AS_STRING(code_obj->co_code);
PyObject *co_code = _PyCode_GetCode(code_obj);
if (co_code == NULL) {
return NULL;
}
_Py_CODEUNIT *code = (_Py_CODEUNIT *)PyBytes_AS_STRING(co_code);
int64_t *stacks = PyMem_New(int64_t, len+1);
int i, j, opcode;
if (stacks == NULL) {
PyErr_NoMemory();
Py_DECREF(co_code);
return NULL;
}
for (int i = 1; i <= len; i++) {
@ -304,6 +309,7 @@ mark_stacks(PyCodeObject *code_obj, int len)
}
}
}
Py_DECREF(co_code);
return stacks;
}
@ -493,7 +499,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore
/* PyCode_NewWithPosOnlyArgs limits co_code to be under INT_MAX so this
* should never overflow. */
int len = (int)(PyBytes_GET_SIZE(f->f_frame->f_code->co_code) / sizeof(_Py_CODEUNIT));
int len = (int)Py_SIZE(f->f_frame->f_code);
int *lines = marklines(f->f_frame->f_code, len);
if (lines == NULL) {
return -1;
@ -838,12 +844,23 @@ PyFrame_New(PyThreadState *tstate, PyCodeObject *code,
static int
_PyFrame_OpAlreadyRan(_PyInterpreterFrame *frame, int opcode, int oparg)
{
const _Py_CODEUNIT *code =
(const _Py_CODEUNIT *)PyBytes_AS_STRING(frame->f_code->co_code);
// This only works when opcode is a non-quickened form:
assert(_PyOpcode_Deopt[opcode] == opcode);
int check_oparg = 0;
for (int i = 0; i < frame->f_lasti; i++) {
if (_Py_OPCODE(code[i]) == opcode && _Py_OPARG(code[i]) == oparg) {
_Py_CODEUNIT instruction = _PyCode_CODE(frame->f_code)[i];
int check_opcode = _PyOpcode_Deopt[_Py_OPCODE(instruction)];
check_oparg |= _Py_OPARG(instruction);
if (check_opcode == opcode && check_oparg == oparg) {
return 1;
}
if (check_opcode == EXTENDED_ARG) {
check_oparg <<= 8;
}
else {
check_oparg = 0;
}
i += _PyOpcode_Caches[check_opcode];
}
return 0;
}
@ -862,7 +879,10 @@ _PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame) {
}
co = frame->f_code;
fast = _PyFrame_GetLocalsArray(frame);
if (frame->f_lasti < 0 && _Py_OPCODE(co->co_firstinstr[0]) == COPY_FREE_VARS) {
// COPY_FREE_VARS has no quickened forms, so no need to use _PyOpcode_Deopt
// here:
if (frame->f_lasti < 0 && _Py_OPCODE(_PyCode_CODE(co)[0]) == COPY_FREE_VARS)
{
/* Free vars have not been initialized -- Do that */
PyCodeObject *co = frame->f_code;
PyObject *closure = frame->f_func->func_closure;
@ -872,6 +892,7 @@ _PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame) {
Py_INCREF(o);
frame->localsplus[offset + i] = o;
}
// COPY_FREE_VARS doesn't have inline CACHEs, either:
frame->f_lasti = 0;
}
for (int i = 0; i < co->co_nlocalsplus; i++) {

19
Objects/genobject.c
View File

@ -349,19 +349,17 @@ _PyGen_yf(PyGenObject *gen)
if (gen->gi_frame_valid) {
_PyInterpreterFrame *frame = (_PyInterpreterFrame *)gen->gi_iframe;
PyObject *bytecode = gen->gi_code->co_code;
unsigned char *code = (unsigned char *)PyBytes_AS_STRING(bytecode);
if (frame->f_lasti < 1) {
/* Return immediately if the frame didn't start yet. SEND
always come after LOAD_CONST: a code object should not start
with SEND */
assert(code[0] != SEND);
assert(_Py_OPCODE(_PyCode_CODE(gen->gi_code)[0]) != SEND);
return NULL;
}
int opcode = code[(frame->f_lasti+1)*sizeof(_Py_CODEUNIT)];
int oparg = code[(frame->f_lasti+1)*sizeof(_Py_CODEUNIT)+1];
if (opcode != RESUME || oparg < 2) {
_Py_CODEUNIT next = _PyCode_CODE(gen->gi_code)[frame->f_lasti + 1];
if (_PyOpcode_Deopt[_Py_OPCODE(next)] != RESUME || _Py_OPARG(next) < 2)
{
/* Not in a yield from */
return NULL;
}
@ -485,14 +483,15 @@ _gen_throw(PyGenObject *gen, int close_on_genexit,
ret = _PyFrame_StackPop((_PyInterpreterFrame *)gen->gi_iframe);
assert(ret == yf);
Py_DECREF(ret);
// XXX: Performing this jump ourselves is awkward and problematic.
// See https://github.com/python/cpython/pull/31968.
/* Termination repetition of SEND loop */
assert(frame->f_lasti >= 0);
PyObject *bytecode = gen->gi_code->co_code;
unsigned char *code = (unsigned char *)PyBytes_AS_STRING(bytecode);
_Py_CODEUNIT *code = _PyCode_CODE(gen->gi_code);
/* Backup to SEND */
frame->f_lasti--;
assert(code[frame->f_lasti*sizeof(_Py_CODEUNIT)] == SEND);
int jump = code[frame->f_lasti*sizeof(_Py_CODEUNIT)+1];
assert(_Py_OPCODE(code[frame->f_lasti]) == SEND);
int jump = _Py_OPARG(code[frame->f_lasti]);
frame->f_lasti += jump;
if (_PyGen_FetchStopIterationValue(&val) == 0) {
ret = gen_send(gen, val);

5
Objects/typeobject.c
View File

@ -8949,7 +8949,10 @@ super_init_without_args(_PyInterpreterFrame *cframe, PyCodeObject *co,
// "firstarg" is a cell here unless (very unlikely) super()
// was called from the C-API before the first MAKE_CELL op.
if (cframe->f_lasti >= 0) {
assert(_Py_OPCODE(*co->co_firstinstr) == MAKE_CELL || _Py_OPCODE(*co->co_firstinstr) == COPY_FREE_VARS);
// MAKE_CELL and COPY_FREE_VARS have no quickened forms, so no need
// to use _PyOpcode_Deopt here:
assert(_Py_OPCODE(_PyCode_CODE(co)[0]) == MAKE_CELL ||
_Py_OPCODE(_PyCode_CODE(co)[0]) == COPY_FREE_VARS);
assert(PyCell_Check(firstarg));
firstarg = PyCell_GET(firstarg);
}

39
Python/ceval.c
View File

@ -1327,9 +1327,8 @@ eval_frame_handle_pending(PyThreadState *tstate)
/* Get opcode and oparg from original instructions, not quickened form. */
#define TRACING_NEXTOPARG() do { \
_Py_CODEUNIT word = ((_Py_CODEUNIT *)PyBytes_AS_STRING(frame->f_code->co_code))[INSTR_OFFSET()]; \
opcode = _Py_OPCODE(word); \
oparg = _Py_OPARG(word); \
NEXTOPARG(); \
opcode = _PyOpcode_Deopt[opcode]; \
} while (0)
/* OpCode prediction macros
@ -1650,9 +1649,10 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
PyCodeObject *co = frame->f_code; \
names = co->co_names; \
consts = co->co_consts; \
first_instr = co->co_firstinstr; \
first_instr = _PyCode_CODE(co); \
} \
assert(frame->f_lasti >= -1); \
/* Jump back to the last instruction executed... */ \
next_instr = first_instr + frame->f_lasti + 1; \
stack_pointer = _PyFrame_GetStackPointer(frame); \
/* Set stackdepth to -1. \
@ -1722,16 +1722,7 @@ handle_eval_breaker:
}
TARGET(RESUME) {
int err = _Py_IncrementCountAndMaybeQuicken(frame->f_code);
if (err) {
if (err < 0) {
goto error;
}
/* Update first_instr and next_instr to point to newly quickened code */
int nexti = INSTR_OFFSET();
first_instr = frame->f_code->co_firstinstr;
next_instr = first_instr + nexti;
}
_PyCode_Warmup(frame->f_code);
JUMP_TO_INSTRUCTION(RESUME_QUICK);
}
@ -4067,16 +4058,7 @@ handle_eval_breaker:
TARGET(JUMP_ABSOLUTE) {
PREDICTED(JUMP_ABSOLUTE);
int err = _Py_IncrementCountAndMaybeQuicken(frame->f_code);
if (err) {
if (err < 0) {
goto error;
}
/* Update first_instr and next_instr to point to newly quickened code */
int nexti = INSTR_OFFSET();
first_instr = frame->f_code->co_firstinstr;
next_instr = first_instr + nexti;
}
_PyCode_Warmup(frame->f_code);
JUMP_TO_INSTRUCTION(JUMP_ABSOLUTE_QUICK);
}
@ -5425,6 +5407,7 @@ handle_eval_breaker:
}
TARGET(EXTENDED_ARG) {
assert(oparg);
int oldoparg = oparg;
NEXTOPARG();
oparg |= oldoparg << 8;
@ -6739,8 +6722,8 @@ maybe_call_line_trace(Py_tracefunc func, PyObject *obj,
*/
initialize_trace_info(&tstate->trace_info, frame);
int entry_point = 0;
_Py_CODEUNIT *code = (_Py_CODEUNIT *)PyBytes_AS_STRING(frame->f_code->co_code);
while (_Py_OPCODE(code[entry_point]) != RESUME) {
_Py_CODEUNIT *code = _PyCode_CODE(frame->f_code);
while (_PyOpcode_Deopt[_Py_OPCODE(code[entry_point])] != RESUME) {
entry_point++;
}
int lastline;
@ -6759,7 +6742,9 @@ maybe_call_line_trace(Py_tracefunc func, PyObject *obj,
/* Trace backward edges (except in 'yield from') or if line number has changed */
int trace = line != lastline ||
(frame->f_lasti < instr_prev &&
_Py_OPCODE(frame->f_code->co_firstinstr[frame->f_lasti]) != SEND);
// SEND has no quickened forms, so no need to use _PyOpcode_Deopt
// here:
_Py_OPCODE(_PyCode_CODE(frame->f_code)[frame->f_lasti]) != SEND);
if (trace) {
result = call_trace(func, obj, tstate, frame, PyTrace_LINE, Py_None);
}

8
Python/compile.c
View File

@ -134,9 +134,9 @@ static int
instr_size(struct instr *instruction)
{
int opcode = instruction->i_opcode;
int oparg = instruction->i_oparg;
int oparg = HAS_ARG(opcode) ? instruction->i_oparg : 0;
int extended_args = (0xFFFFFF < oparg) + (0xFFFF < oparg) + (0xFF < oparg);
int caches = _PyOpcode_InlineCacheEntries[opcode];
int caches = _PyOpcode_Caches[opcode];
return extended_args + 1 + caches;
}
@ -144,8 +144,8 @@ static void
write_instr(_Py_CODEUNIT *codestr, struct instr *instruction, int ilen)
{
int opcode = instruction->i_opcode;
int oparg = instruction->i_oparg;
int caches = _PyOpcode_InlineCacheEntries[opcode];
int oparg = HAS_ARG(opcode) ? instruction->i_oparg : 0;
int caches = _PyOpcode_Caches[opcode];
switch (ilen - caches) {
case 4:
*codestr++ = _Py_MAKECODEUNIT(EXTENDED_ARG, (oparg >> 24) & 0xFF);

8
Python/marshal.c
View File

@ -544,13 +544,18 @@ w_complex_object(PyObject *v, char flag, WFILE *p)
}
else if (PyCode_Check(v)) {
PyCodeObject *co = (PyCodeObject *)v;
PyObject *co_code = _PyCode_GetCode(co);
if (co_code == NULL) {
p->error = WFERR_NOMEMORY;
return;
}
W_TYPE(TYPE_CODE, p);
w_long(co->co_argcount, p);
w_long(co->co_posonlyargcount, p);
w_long(co->co_kwonlyargcount, p);
w_long(co->co_stacksize, p);
w_long(co->co_flags, p);
w_object(co->co_code, p);
w_object(co_code, p);
w_object(co->co_consts, p);
w_object(co->co_names, p);
w_object(co->co_localsplusnames, p);
@ -563,6 +568,7 @@ w_complex_object(PyObject *v, char flag, WFILE *p)
w_object(co->co_endlinetable, p);
w_object(co->co_columntable, p);
w_object(co->co_exceptiontable, p);
Py_DECREF(co_code);
}
else if (PyObject_CheckBuffer(v)) {
/* Write unknown bytes-like objects as a bytes object */

72
Python/opcode_targets.h generated
View File

@ -3,49 +3,49 @@ static void *opcode_targets[256] = {
&&TARGET_POP_TOP,
&&TARGET_PUSH_NULL,
&&TARGET_BINARY_OP_ADAPTIVE,
&&TARGET_BINARY_OP_ADD_INT,
&&TARGET_BINARY_OP_ADD_FLOAT,
&&TARGET_BINARY_OP_ADD_INT,
&&TARGET_BINARY_OP_ADD_UNICODE,
&&TARGET_BINARY_OP_INPLACE_ADD_UNICODE,
&&TARGET_BINARY_OP_MULTIPLY_INT,
&&TARGET_BINARY_OP_MULTIPLY_FLOAT,
&&TARGET_NOP,
&&TARGET_UNARY_POSITIVE,
&&TARGET_UNARY_NEGATIVE,
&&TARGET_UNARY_NOT,
&&TARGET_BINARY_OP_MULTIPLY_FLOAT,
&&TARGET_BINARY_OP_SUBTRACT_INT,
&&TARGET_UNARY_INVERT,
&&TARGET_BINARY_OP_MULTIPLY_INT,
&&TARGET_BINARY_OP_SUBTRACT_FLOAT,
&&TARGET_UNARY_INVERT,
&&TARGET_BINARY_OP_SUBTRACT_INT,
&&TARGET_BINARY_SUBSCR_ADAPTIVE,
&&TARGET_BINARY_SUBSCR_DICT,
&&TARGET_BINARY_SUBSCR_GETITEM,
&&TARGET_BINARY_SUBSCR_LIST_INT,
&&TARGET_BINARY_SUBSCR_TUPLE_INT,
&&TARGET_CALL_ADAPTIVE,
&&TARGET_CALL_PY_EXACT_ARGS,
&&TARGET_CALL_PY_WITH_DEFAULTS,
&&TARGET_BINARY_SUBSCR,
&&TARGET_COMPARE_OP_ADAPTIVE,
&&TARGET_COMPARE_OP_FLOAT_JUMP,
&&TARGET_COMPARE_OP_INT_JUMP,
&&TARGET_COMPARE_OP_STR_JUMP,
&&TARGET_BINARY_SUBSCR_ADAPTIVE,
&&TARGET_BINARY_SUBSCR_GETITEM,
&&TARGET_BINARY_SUBSCR_LIST_INT,
&&TARGET_BINARY_SUBSCR_TUPLE_INT,
&&TARGET_BINARY_SUBSCR,
&&TARGET_BINARY_SUBSCR_DICT,
&&TARGET_STORE_SUBSCR_ADAPTIVE,
&&TARGET_STORE_SUBSCR_LIST_INT,
&&TARGET_STORE_SUBSCR_DICT,
&&TARGET_GET_LEN,
&&TARGET_MATCH_MAPPING,
&&TARGET_MATCH_SEQUENCE,
&&TARGET_MATCH_KEYS,
&&TARGET_CALL_ADAPTIVE,
&&TARGET_PUSH_EXC_INFO,
&&TARGET_CALL_PY_EXACT_ARGS,
&&TARGET_CALL_PY_WITH_DEFAULTS,
&&TARGET_JUMP_ABSOLUTE_QUICK,
&&TARGET_PUSH_EXC_INFO,
&&TARGET_LOAD_ATTR_ADAPTIVE,
&&TARGET_LOAD_ATTR_INSTANCE_VALUE,
&&TARGET_LOAD_ATTR_WITH_HINT,
&&TARGET_LOAD_ATTR_SLOT,
&&TARGET_LOAD_ATTR_MODULE,
&&TARGET_LOAD_ATTR_SLOT,
&&TARGET_LOAD_ATTR_WITH_HINT,
&&TARGET_LOAD_CONST__LOAD_FAST,
&&TARGET_LOAD_FAST__LOAD_CONST,
&&TARGET_LOAD_FAST__LOAD_FAST,
&&TARGET_LOAD_GLOBAL_ADAPTIVE,
&&TARGET_LOAD_GLOBAL_MODULE,
&&TARGET_LOAD_GLOBAL_BUILTIN,
&&TARGET_LOAD_GLOBAL_MODULE,
&&TARGET_LOAD_METHOD_ADAPTIVE,
&&TARGET_LOAD_METHOD_CLASS,
&&TARGET_WITH_EXCEPT_START,
@ -61,26 +61,26 @@ static void *opcode_targets[256] = {
&&TARGET_PRECALL_ADAPTIVE,
&&TARGET_STORE_SUBSCR,
&&TARGET_DELETE_SUBSCR,
&&TARGET_PRECALL_BOUND_METHOD,
&&TARGET_PRECALL_BUILTIN_CLASS,
&&TARGET_PRECALL_NO_KW_BUILTIN_O,
&&TARGET_PRECALL_NO_KW_BUILTIN_FAST,
&&TARGET_PRECALL_BUILTIN_FAST_WITH_KEYWORDS,
&&TARGET_PRECALL_NO_KW_LEN,
&&TARGET_PRECALL_NO_KW_BUILTIN_FAST,
&&TARGET_PRECALL_NO_KW_BUILTIN_O,
&&TARGET_PRECALL_NO_KW_ISINSTANCE,
&&TARGET_GET_ITER,
&&TARGET_GET_YIELD_FROM_ITER,
&&TARGET_PRINT_EXPR,
&&TARGET_LOAD_BUILD_CLASS,
&&TARGET_PRECALL_NO_KW_LEN,
&&TARGET_PRECALL_NO_KW_LIST_APPEND,
&&TARGET_PRECALL_NO_KW_METHOD_DESCRIPTOR_O,
&&TARGET_LOAD_ASSERTION_ERROR,
&&TARGET_RETURN_GENERATOR,
&&TARGET_PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST,
&&TARGET_PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS,
&&TARGET_PRECALL_NO_KW_METHOD_DESCRIPTOR_O,
&&TARGET_PRECALL_NO_KW_STR_1,
&&TARGET_PRECALL_NO_KW_TUPLE_1,
&&TARGET_PRECALL_NO_KW_TYPE_1,
&&TARGET_PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST,
&&TARGET_PRECALL_BOUND_METHOD,
&&TARGET_LIST_TO_TUPLE,
&&TARGET_RETURN_VALUE,
&&TARGET_IMPORT_STAR,
@ -157,23 +157,23 @@ static void *opcode_targets[256] = {
&&TARGET_FORMAT_VALUE,
&&TARGET_BUILD_CONST_KEY_MAP,
&&TARGET_BUILD_STRING,
&&TARGET_UNPACK_SEQUENCE_ADAPTIVE,
&&TARGET_UNPACK_SEQUENCE_LIST,
&&TARGET_STORE_FAST__LOAD_FAST,
&&TARGET_STORE_FAST__STORE_FAST,
&&TARGET_LOAD_METHOD,
&&TARGET_UNPACK_SEQUENCE_TUPLE,
&&TARGET_STORE_SUBSCR_ADAPTIVE,
&&TARGET_LIST_EXTEND,
&&TARGET_SET_UPDATE,
&&TARGET_DICT_MERGE,
&&TARGET_DICT_UPDATE,
&&TARGET_PRECALL,
&&TARGET_UNPACK_SEQUENCE_TWO_TUPLE,
&&TARGET_LOAD_FAST__LOAD_FAST,
&&TARGET_STORE_FAST__LOAD_FAST,
&&TARGET_LOAD_FAST__LOAD_CONST,
&&TARGET_STORE_SUBSCR_DICT,
&&TARGET_STORE_SUBSCR_LIST_INT,
&&TARGET_UNPACK_SEQUENCE_ADAPTIVE,
&&TARGET_UNPACK_SEQUENCE_LIST,
&&TARGET_CALL,
&&TARGET_KW_NAMES,
&&TARGET_LOAD_CONST__LOAD_FAST,
&&TARGET_STORE_FAST__STORE_FAST,
&&TARGET_UNPACK_SEQUENCE_TUPLE,
&&TARGET_UNPACK_SEQUENCE_TWO_TUPLE,
&&_unknown_opcode,
&&_unknown_opcode,
&&_unknown_opcode,

100
Python/specialize.c
View File

@ -15,31 +15,6 @@
* ./adaptive.md
*/
/* We layout the quickened data as a bi-directional array:
* Instructions upwards, cache entries downwards.
* first_instr is aligned to a SpecializedCacheEntry.
* The nth instruction is located at first_instr[n]
* The nth cache is located at ((SpecializedCacheEntry *)first_instr)[-1-n]
* The first (index 0) cache entry is reserved for the count, to enable finding
* the first instruction from the base pointer.
* The cache_count argument must include space for the count.
* We use the SpecializedCacheOrInstruction union to refer to the data
* to avoid type punning.
Layout of quickened data, each line 8 bytes for M cache entries and N instructions:
<cache_count> <---- co->co_quickened
<cache M-1>
<cache M-2>
...
<cache 0>
<instr 0> <instr 1> <instr 2> <instr 3> <--- co->co_first_instr
<instr 4> <instr 5> <instr 6> <instr 7>
...
<instr N-1>
*/
/* Map from opcode to adaptive opcode.
Values of zero are ignored. */
static uint8_t adaptive_opcodes[256] = {
@ -275,26 +250,14 @@ _Py_PrintSpecializationStats(int to_file)
#define SPECIALIZATION_FAIL(opcode, kind) ((void)0)
#endif
static _Py_CODEUNIT *
allocate(int instruction_count)
// Insert adaptive instructions and superinstructions. This cannot fail.
void
_PyCode_Quicken(PyCodeObject *code)
{
assert(instruction_count > 0);
void *array = PyMem_Malloc(sizeof(_Py_CODEUNIT) * instruction_count);
if (array == NULL) {
PyErr_NoMemory();
return NULL;
}
_Py_QuickenedCount++;
return (_Py_CODEUNIT *)array;
}
// Insert adaptive instructions and superinstructions.
static void
optimize(_Py_CODEUNIT *instructions, int len)
{
int previous_opcode = -1;
for(int i = 0; i < len; i++) {
_Py_CODEUNIT *instructions = _PyCode_CODE(code);
for (int i = 0; i < Py_SIZE(code); i++) {
int opcode = _Py_OPCODE(instructions[i]);
uint8_t adaptive_opcode = adaptive_opcodes[opcode];
if (adaptive_opcode) {
@ -302,10 +265,10 @@ optimize(_Py_CODEUNIT *instructions, int len)
// Make sure the adaptive counter is zero:
assert(instructions[i + 1] == 0);
previous_opcode = -1;
i += _PyOpcode_InlineCacheEntries[opcode];
i += _PyOpcode_Caches[opcode];
}
else {
assert(!_PyOpcode_InlineCacheEntries[opcode]);
assert(!_PyOpcode_Caches[opcode]);
switch (opcode) {
case JUMP_ABSOLUTE:
_Py_SET_OPCODE(instructions[i], JUMP_ABSOLUTE_QUICK);
@ -347,28 +310,6 @@ optimize(_Py_CODEUNIT *instructions, int len)
}
}
int
_Py_Quicken(PyCodeObject *code) {
if (code->co_quickened) {
return 0;
}
Py_ssize_t size = PyBytes_GET_SIZE(code->co_code);
int instr_count = (int)(size/sizeof(_Py_CODEUNIT));
if (instr_count > MAX_SIZE_TO_QUICKEN) {
code->co_warmup = QUICKENING_WARMUP_COLDEST;
return 0;
}
_Py_CODEUNIT *quickened = allocate(instr_count);
if (quickened == NULL) {
return -1;
}
memcpy(quickened, code->co_firstinstr, size);
optimize(quickened, instr_count);
code->co_quickened = quickened;
code->co_firstinstr = quickened;
return 0;
}
static inline int
initial_counter_value(void) {
/* Starting value for the counter.
@ -705,8 +646,7 @@ specialize_dict_access(
int
_Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name)
{
assert(_PyOpcode_InlineCacheEntries[LOAD_ATTR] ==
INLINE_CACHE_ENTRIES_LOAD_ATTR);
assert(_PyOpcode_Caches[LOAD_ATTR] == INLINE_CACHE_ENTRIES_LOAD_ATTR);
_PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
if (PyModule_CheckExact(owner)) {
int err = specialize_module_load_attr(owner, instr, name, LOAD_ATTR,
@ -804,8 +744,7 @@ success:
int
_Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name)
{
assert(_PyOpcode_InlineCacheEntries[STORE_ATTR] ==
INLINE_CACHE_ENTRIES_STORE_ATTR);
assert(_PyOpcode_Caches[STORE_ATTR] == INLINE_CACHE_ENTRIES_STORE_ATTR);
_PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
PyTypeObject *type = Py_TYPE(owner);
if (PyModule_CheckExact(owner)) {
@ -965,8 +904,7 @@ typedef enum {
int
_Py_Specialize_LoadMethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name)
{
assert(_PyOpcode_InlineCacheEntries[LOAD_METHOD] ==
INLINE_CACHE_ENTRIES_LOAD_METHOD);
assert(_PyOpcode_Caches[LOAD_METHOD] == INLINE_CACHE_ENTRIES_LOAD_METHOD);
_PyLoadMethodCache *cache = (_PyLoadMethodCache *)(instr + 1);
PyTypeObject *owner_cls = Py_TYPE(owner);
@ -1098,8 +1036,7 @@ _Py_Specialize_LoadGlobal(
PyObject *globals, PyObject *builtins,
_Py_CODEUNIT *instr, PyObject *name)
{
assert(_PyOpcode_InlineCacheEntries[LOAD_GLOBAL] ==
INLINE_CACHE_ENTRIES_LOAD_GLOBAL);
assert(_PyOpcode_Caches[LOAD_GLOBAL] == INLINE_CACHE_ENTRIES_LOAD_GLOBAL);
/* Use inline cache */
_PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)(instr + 1);
assert(PyUnicode_CheckExact(name));
@ -1235,7 +1172,7 @@ int
_Py_Specialize_BinarySubscr(
PyObject *container, PyObject *sub, _Py_CODEUNIT *instr)
{
assert(_PyOpcode_InlineCacheEntries[BINARY_SUBSCR] ==
assert(_PyOpcode_Caches[BINARY_SUBSCR] ==
INLINE_CACHE_ENTRIES_BINARY_SUBSCR);
_PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)(instr + 1);
PyTypeObject *container_type = Py_TYPE(container);
@ -1663,8 +1600,7 @@ int
_Py_Specialize_Precall(PyObject *callable, _Py_CODEUNIT *instr, int nargs,
PyObject *kwnames, int oparg)
{
assert(_PyOpcode_InlineCacheEntries[PRECALL] ==
INLINE_CACHE_ENTRIES_PRECALL);
assert(_PyOpcode_Caches[PRECALL] == INLINE_CACHE_ENTRIES_PRECALL);
_PyPrecallCache *cache = (_PyPrecallCache *)(instr + 1);
int fail;
if (PyCFunction_CheckExact(callable)) {
@ -1710,7 +1646,7 @@ int
_Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs,
PyObject *kwnames)
{
assert(_PyOpcode_InlineCacheEntries[CALL] == INLINE_CACHE_ENTRIES_CALL);
assert(_PyOpcode_Caches[CALL] == INLINE_CACHE_ENTRIES_CALL);
_PyCallCache *cache = (_PyCallCache *)(instr + 1);
int fail;
if (PyFunction_Check(callable)) {
@ -1808,8 +1744,7 @@ void
_Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
int oparg)
{
assert(_PyOpcode_InlineCacheEntries[BINARY_OP] ==
INLINE_CACHE_ENTRIES_BINARY_OP);
assert(_PyOpcode_Caches[BINARY_OP] == INLINE_CACHE_ENTRIES_BINARY_OP);
_PyBinaryOpCache *cache = (_PyBinaryOpCache *)(instr + 1);
switch (oparg) {
case NB_ADD:
@ -1936,8 +1871,7 @@ void
_Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
int oparg)
{
assert(_PyOpcode_InlineCacheEntries[COMPARE_OP] ==
INLINE_CACHE_ENTRIES_COMPARE_OP);
assert(_PyOpcode_Caches[COMPARE_OP] == INLINE_CACHE_ENTRIES_COMPARE_OP);
_PyCompareOpCache *cache = (_PyCompareOpCache *)(instr + 1);
int next_opcode = _Py_OPCODE(instr[INLINE_CACHE_ENTRIES_COMPARE_OP + 1]);
if (next_opcode != POP_JUMP_IF_FALSE && next_opcode != POP_JUMP_IF_TRUE) {
@ -2019,7 +1953,7 @@ unpack_sequence_fail_kind(PyObject *seq)
void
_Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr, int oparg)
{
assert(_PyOpcode_InlineCacheEntries[UNPACK_SEQUENCE] ==
assert(_PyOpcode_Caches[UNPACK_SEQUENCE] ==
INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE);
_PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)(instr + 1);
if (PyTuple_CheckExact(seq)) {

35
Tools/scripts/deepfreeze.py
View File

@ -229,12 +229,8 @@ class Printer:
def generate_code(self, name: str, code: types.CodeType) -> str:
# The ordering here matches PyCode_NewWithPosOnlyArgs()
# (but see below).
co_code = self.generate(name + "_code", code.co_code)
co_consts = self.generate(name + "_consts", code.co_consts)
co_names = self.generate(name + "_names", code.co_names)
co_varnames = self.generate(name + "_varnames", code.co_varnames)
co_freevars = self.generate(name + "_freevars", code.co_freevars)
co_cellvars = self.generate(name + "_cellvars", code.co_cellvars)
co_filename = self.generate(name + "_filename", code.co_filename)
co_name = self.generate(name + "_name", code.co_name)
co_qualname = self.generate(name + "_qualname", code.co_qualname)
@ -249,14 +245,17 @@ class Printer:
# Derived values
nlocals, nplaincellvars, ncellvars, nfreevars = \
get_localsplus_counts(code, localsplusnames, localspluskinds)
with self.block(f"static struct PyCodeObject {name} =", ";"):
self.object_head("PyCode_Type")
co_code_adaptive = make_string_literal(code.co_code)
self.write("static")
with self.indent():
self.write(f"struct _PyCode_DEF({len(code.co_code)})")
with self.block(f"{name} =", ";"):
self.object_var_head("PyCode_Type", len(code.co_code) // 2)
# But the ordering here must match that in cpython/code.h
# (which is a pain because we tend to reorder those for perf)
# otherwise MSVC doesn't like it.
self.write(f".co_consts = {co_consts},")
self.write(f".co_names = {co_names},")
self.write(f".co_firstinstr = (_Py_CODEUNIT *) {removesuffix(co_code, '.ob_base.ob_base')}.ob_sval,")
self.write(f".co_exceptiontable = {co_exceptiontable},")
self.field(code, "co_flags")
self.write(".co_warmup = QUICKENING_INITIAL_WARMUP_VALUE,")
@ -265,7 +264,11 @@ class Printer:
self.field(code, "co_kwonlyargcount")
self.field(code, "co_stacksize")
self.field(code, "co_firstlineno")
self.write(f".co_code = {co_code},")
self.write(f".co_nlocalsplus = {len(localsplusnames)},")
self.field(code, "co_nlocals")
self.write(f".co_nplaincellvars = {nplaincellvars},")
self.write(f".co_ncellvars = {ncellvars},")
self.write(f".co_nfreevars = {nfreevars},")
self.write(f".co_localsplusnames = {co_localsplusnames},")
self.write(f".co_localspluskinds = {co_localspluskinds},")
self.write(f".co_filename = {co_filename},")
@ -274,17 +277,11 @@ class Printer:
self.write(f".co_linetable = {co_linetable},")
self.write(f".co_endlinetable = {co_endlinetable},")
self.write(f".co_columntable = {co_columntable},")
self.write(f".co_nlocalsplus = {len(localsplusnames)},")
self.field(code, "co_nlocals")
self.write(f".co_nplaincellvars = {nplaincellvars},")
self.write(f".co_ncellvars = {ncellvars},")
self.write(f".co_nfreevars = {nfreevars},")
self.write(f".co_varnames = {co_varnames},")
self.write(f".co_cellvars = {co_cellvars},")
self.write(f".co_freevars = {co_freevars},")
self.deallocs.append(f"_PyStaticCode_Dealloc(&{name});")
self.interns.append(f"_PyStaticCode_InternStrings(&{name})")
return f"& {name}.ob_base"
self.write(f".co_code_adaptive = {co_code_adaptive},")
name_as_code = f"(PyCodeObject *)&{name}"
self.deallocs.append(f"_PyStaticCode_Dealloc({name_as_code});")
self.interns.append(f"_PyStaticCode_InternStrings({name_as_code})")
return f"& {name}.ob_base.ob_base"
def generate_tuple(self, name: str, t: Tuple[object, ...]) -> str:
if len(t) == 0:

17
Tools/scripts/generate_opcode_h.py
View File

@ -28,7 +28,7 @@ footer = """
#endif /* !Py_OPCODE_H */
"""
DEFINE = "#define {:<31} {:>3}\n"
DEFINE = "#define {:<38} {:>3}\n"
UINT32_MASK = (1<<32)-1
@ -75,16 +75,27 @@ def main(opcode_py, outfile='Include/opcode.h'):
fobj.write(DEFINE.format(name, next_op))
used[next_op] = True
fobj.write(DEFINE.format('DO_TRACING', 255))
fobj.write("\nextern const uint8_t _PyOpcode_InlineCacheEntries[256];\n")
fobj.write("\nextern const uint8_t _PyOpcode_Caches[256];\n")
fobj.write("\nextern const uint8_t _PyOpcode_Deopt[256];\n")
fobj.write("\n#ifdef NEED_OPCODE_TABLES\n")
write_int_array_from_ops("_PyOpcode_RelativeJump", opcode['hasjrel'], fobj)
write_int_array_from_ops("_PyOpcode_Jump", opcode['hasjrel'] + opcode['hasjabs'], fobj)
fobj.write("\nconst uint8_t _PyOpcode_InlineCacheEntries[256] = {\n")
fobj.write("\nconst uint8_t _PyOpcode_Caches[256] = {\n")
for i, entries in enumerate(opcode["_inline_cache_entries"]):
if entries:
fobj.write(f" [{opname[i]}] = {entries},\n")
fobj.write("};\n")
deoptcodes = {}
for basic in opmap:
deoptcodes[basic] = basic
for basic, family in opcode["_specializations"].items():
for specialized in family:
deoptcodes[specialized] = basic
fobj.write("\nconst uint8_t _PyOpcode_Deopt[256] = {\n")
for opt, deopt in sorted(deoptcodes.items()):
fobj.write(f" [{opt}] = {deopt},\n")
fobj.write("};\n")
fobj.write("#endif /* OPCODE_TABLES */\n")
fobj.write("\n")