diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h
index e729904ff2c..ea5b99ebba0 100644
--- a/Include/internal/pycore_ceval.h
+++ b/Include/internal/pycore_ceval.h
@@ -158,6 +158,18 @@ extern int _Py_HandlePending(PyThreadState *tstate);
extern PyObject * _PyEval_GetFrameLocals(void);
+extern const binaryfunc _PyEval_BinaryOps[];
+int _PyEval_CheckExceptStarTypeValid(PyThreadState *tstate, PyObject* right);
+int _PyEval_CheckExceptTypeValid(PyThreadState *tstate, PyObject* right);
+int _PyEval_ExceptionGroupMatch(PyObject* exc_value, PyObject *match_type, PyObject **match, PyObject **rest);
+void _PyEval_FormatAwaitableError(PyThreadState *tstate, PyTypeObject *type, int oparg);
+void _PyEval_FormatExcCheckArg(PyThreadState *tstate, PyObject *exc, const char *format_str, PyObject *obj);
+void _PyEval_FormatExcUnbound(PyThreadState *tstate, PyCodeObject *co, int oparg);
+void _PyEval_FormatKwargsError(PyThreadState *tstate, PyObject *func, PyObject *kwargs);
+PyObject *_PyEval_MatchClass(PyThreadState *tstate, PyObject *subject, PyObject *type, Py_ssize_t nargs, PyObject *kwargs);
+PyObject *_PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys);
+int _PyEval_UnpackIterable(PyThreadState *tstate, PyObject *v, int argcnt, int argcntafter, PyObject **sp);
+
#ifdef __cplusplus
}
diff --git a/Makefile.pre.in b/Makefile.pre.in
index f4f6ff595b0..f3a0db163ed 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -381,6 +381,7 @@ PYTHON_OBJS= \
Python/context.o \
Python/dynamic_annotations.o \
Python/errors.o \
+ Python/executor.o \
Python/flowgraph.o \
Python/frame.o \
Python/frozenmain.o \
@@ -1562,10 +1563,13 @@ Python/ceval.o: \
$(srcdir)/Python/ceval_macros.h \
$(srcdir)/Python/condvar.h \
$(srcdir)/Python/generated_cases.c.h \
- $(srcdir)/Python/executor_cases.c.h \
- $(srcdir)/Include/internal/pycore_opcode_metadata.h \
$(srcdir)/Python/opcode_targets.h
+Python/executor.o: \
+ $(srcdir)/Include/internal/pycore_opcode_metadata.h \
+ $(srcdir)/Python/ceval_macros.h \
+ $(srcdir)/Python/executor_cases.c.h
+
Python/flowgraph.o: \
$(srcdir)/Include/internal/pycore_opcode_metadata.h
diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj
index 87f08e857d0..e247637a0df 100644
--- a/PCbuild/_freeze_module.vcxproj
+++ b/PCbuild/_freeze_module.vcxproj
@@ -192,6 +192,7 @@
+
diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters
index 3dae8cca7a2..2a0e0093080 100644
--- a/PCbuild/_freeze_module.vcxproj.filters
+++ b/PCbuild/_freeze_module.vcxproj.filters
@@ -124,6 +124,9 @@
Source Files
+
+ Source Files
+
Source Files
diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj
index 760962e4c4b..0fe24082ca4 100644
--- a/PCbuild/pythoncore.vcxproj
+++ b/PCbuild/pythoncore.vcxproj
@@ -520,6 +520,7 @@
+
diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters
index aaebe1908e3..2b3793cb0f7 100644
--- a/PCbuild/pythoncore.vcxproj.filters
+++ b/PCbuild/pythoncore.vcxproj.filters
@@ -1145,6 +1145,9 @@
Python
+
+ Python
+
Python
diff --git a/Python/bytecodes.c b/Python/bytecodes.c
index b9434390ea4..3a191aca730 100644
--- a/Python/bytecodes.c
+++ b/Python/bytecodes.c
@@ -77,7 +77,6 @@ dummy_func(
PyObject **stack_pointer,
PyObject *kwnames,
int throwflag,
- binaryfunc binary_ops[],
PyObject *args[]
)
{
@@ -893,7 +892,7 @@ dummy_func(
iter = _PyCoro_GetAwaitableIter(iterable);
if (iter == NULL) {
- format_awaitable_error(tstate, Py_TYPE(iterable), oparg);
+ _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg);
}
DECREF_INPUTS();
@@ -1120,9 +1119,9 @@ dummy_func(
err = PyObject_DelItem(ns, name);
// Can't use ERROR_IF here.
if (err != 0) {
- format_exc_check_arg(tstate, PyExc_NameError,
- NAME_ERROR_MSG,
- name);
+ _PyEval_FormatExcCheckArg(tstate, PyExc_NameError,
+ NAME_ERROR_MSG,
+ name);
goto error;
}
}
@@ -1145,7 +1144,7 @@ dummy_func(
DECREMENT_ADAPTIVE_COUNTER(cache->counter);
#endif /* ENABLE_SPECIALIZATION */
PyObject **top = stack_pointer + oparg - 1;
- int res = unpack_iterable(tstate, seq, oparg, -1, top);
+ int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top);
DECREF_INPUTS();
ERROR_IF(res == 0, error);
}
@@ -1185,7 +1184,7 @@ dummy_func(
inst(UNPACK_EX, (seq -- unused[oparg & 0xFF], unused, unused[oparg >> 8])) {
int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8);
PyObject **top = stack_pointer + totalargs - 1;
- int res = unpack_iterable(tstate, seq, oparg & 0xFF, oparg >> 8, top);
+ int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top);
DECREF_INPUTS();
ERROR_IF(res == 0, error);
}
@@ -1235,8 +1234,8 @@ dummy_func(
// Can't use ERROR_IF here.
if (err != 0) {
if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) {
- format_exc_check_arg(tstate, PyExc_NameError,
- NAME_ERROR_MSG, name);
+ _PyEval_FormatExcCheckArg(tstate, PyExc_NameError,
+ NAME_ERROR_MSG, name);
}
goto error;
}
@@ -1274,7 +1273,7 @@ dummy_func(
goto error;
}
if (v == NULL) {
- format_exc_check_arg(
+ _PyEval_FormatExcCheckArg(
tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
goto error;
@@ -1315,8 +1314,8 @@ dummy_func(
if (!_PyErr_Occurred(tstate)) {
/* _PyDict_LoadGlobal() returns NULL without raising
* an exception if the key doesn't exist */
- format_exc_check_arg(tstate, PyExc_NameError,
- NAME_ERROR_MSG, name);
+ _PyEval_FormatExcCheckArg(tstate, PyExc_NameError,
+ NAME_ERROR_MSG, name);
}
ERROR_IF(true, error);
}
@@ -1331,7 +1330,7 @@ dummy_func(
/* namespace 2: builtins */
ERROR_IF(PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0, error);
if (v == NULL) {
- format_exc_check_arg(
+ _PyEval_FormatExcCheckArg(
tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
ERROR_IF(true, error);
@@ -1413,7 +1412,7 @@ dummy_func(
// Can't use ERROR_IF here.
// Fortunately we don't need its superpower.
if (oldobj == NULL) {
- format_exc_unbound(tstate, _PyFrame_GetCode(frame), oparg);
+ _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
goto error;
}
PyCell_SET(cell, NULL);
@@ -1434,7 +1433,7 @@ dummy_func(
PyObject *cell = GETLOCAL(oparg);
value = PyCell_GET(cell);
if (value == NULL) {
- format_exc_unbound(tstate, _PyFrame_GetCode(frame), oparg);
+ _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
goto error;
}
Py_INCREF(value);
@@ -1445,7 +1444,7 @@ dummy_func(
PyObject *cell = GETLOCAL(oparg);
value = PyCell_GET(cell);
if (value == NULL) {
- format_exc_unbound(tstate, _PyFrame_GetCode(frame), oparg);
+ _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
ERROR_IF(true, error);
}
Py_INCREF(value);
@@ -1612,7 +1611,7 @@ dummy_func(
PyObject *dict = PEEK(oparg + 1); // update is still on the stack
if (_PyDict_MergeEx(dict, update, 2) < 0) {
- format_kwargs_error(tstate, PEEK(3 + oparg), update);
+ _PyEval_FormatKwargsError(tstate, PEEK(3 + oparg), update);
DECREF_INPUTS();
ERROR_IF(true, error);
}
@@ -2126,15 +2125,15 @@ dummy_func(
}
inst(CHECK_EG_MATCH, (exc_value, match_type -- rest, match)) {
- if (check_except_star_type_valid(tstate, match_type) < 0) {
+ if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) {
DECREF_INPUTS();
ERROR_IF(true, error);
}
match = NULL;
rest = NULL;
- int res = exception_group_match(exc_value, match_type,
- &match, &rest);
+ int res = _PyEval_ExceptionGroupMatch(exc_value, match_type,
+ &match, &rest);
DECREF_INPUTS();
ERROR_IF(res < 0, error);
@@ -2148,7 +2147,7 @@ dummy_func(
inst(CHECK_EXC_MATCH, (left, right -- left, b)) {
assert(PyExceptionInstance_Check(left));
- if (check_except_type_valid(tstate, right) < 0) {
+ if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) {
DECREF_INPUTS();
ERROR_IF(true, error);
}
@@ -2275,7 +2274,7 @@ dummy_func(
// Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or
// None on failure.
assert(PyTuple_CheckExact(names));
- attrs = match_class(tstate, subject, type, oparg, names);
+ attrs = _PyEval_MatchClass(tstate, subject, type, oparg, names);
DECREF_INPUTS();
if (attrs) {
assert(PyTuple_CheckExact(attrs)); // Success!
@@ -2298,7 +2297,7 @@ dummy_func(
inst(MATCH_KEYS, (subject, keys -- subject, keys, values_or_none)) {
// On successful match, PUSH(values). Otherwise, PUSH(None).
- values_or_none = match_keys(tstate, subject, keys);
+ values_or_none = _PyEval_MatchKeys(tstate, subject, keys);
ERROR_IF(values_or_none == NULL, error);
}
@@ -3617,10 +3616,10 @@ dummy_func(
STAT_INC(BINARY_OP, deferred);
DECREMENT_ADAPTIVE_COUNTER(cache->counter);
#endif /* ENABLE_SPECIALIZATION */
- assert(0 <= oparg);
- assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops));
- assert(binary_ops[oparg]);
- res = binary_ops[oparg](lhs, rhs);
+ assert(NB_ADD <= oparg);
+ assert(oparg <= NB_INPLACE_XOR);
+ assert(_PyEval_BinaryOps[oparg]);
+ res = _PyEval_BinaryOps[oparg](lhs, rhs);
DECREF_INPUTS();
ERROR_IF(res == NULL, error);
}
diff --git a/Python/ceval.c b/Python/ceval.c
index b56ddfb4bd2..9f2855f964a 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -96,13 +96,6 @@
} while (0)
#endif
-// GH-89279: Similar to above, force inlining by using a macro.
-#if defined(_MSC_VER) && SIZEOF_INT == 4
-#define _Py_atomic_load_relaxed_int32(ATOMIC_VAL) (assert(sizeof((ATOMIC_VAL)->_value) == 4), *((volatile int*)&((ATOMIC_VAL)->_value)))
-#else
-#define _Py_atomic_load_relaxed_int32(ATOMIC_VAL) _Py_atomic_load_relaxed(ATOMIC_VAL)
-#endif
-
#ifdef LLTRACE
static void
@@ -206,13 +199,7 @@ static void monitor_throw(PyThreadState *tstate,
static PyObject * import_name(PyThreadState *, _PyInterpreterFrame *,
PyObject *, PyObject *, PyObject *);
static PyObject * import_from(PyThreadState *, PyObject *, PyObject *);
-static void format_exc_check_arg(PyThreadState *, PyObject *, const char *, PyObject *);
-static void format_exc_unbound(PyThreadState *tstate, PyCodeObject *co, int oparg);
static int check_args_iterable(PyThreadState *, PyObject *func, PyObject *vararg);
-static int check_except_type_valid(PyThreadState *tstate, PyObject* right);
-static int check_except_star_type_valid(PyThreadState *tstate, PyObject* right);
-static void format_kwargs_error(PyThreadState *, PyObject *func, PyObject *kwargs);
-static void format_awaitable_error(PyThreadState *, PyTypeObject *, int);
static int get_exception_handler(PyCodeObject *, int, int*, int*, int*);
static _PyInterpreterFrame *
_PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func,
@@ -224,12 +211,6 @@ _PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func,
static void
_PyEvalFrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame);
-#define UNBOUNDLOCAL_ERROR_MSG \
- "cannot access local variable '%s' where it is not associated with a value"
-#define UNBOUNDFREE_ERROR_MSG \
- "cannot access free variable '%s' where it is not associated with a" \
- " value in enclosing scope"
-
#ifdef HAVE_ERRNO_H
#include
#endif
@@ -286,7 +267,7 @@ _Py_CheckRecursiveCall(PyThreadState *tstate, const char *where)
}
-static const binaryfunc binary_ops[] = {
+const binaryfunc _PyEval_BinaryOps[] = {
[NB_ADD] = PyNumber_Add,
[NB_AND] = PyNumber_And,
[NB_FLOOR_DIVIDE] = PyNumber_FloorDivide,
@@ -321,8 +302,8 @@ static const binaryfunc binary_ops[] = {
// Return a tuple of values corresponding to keys, with error checks for
// duplicate/missing keys.
-static PyObject*
-match_keys(PyThreadState *tstate, PyObject *map, PyObject *keys)
+PyObject *
+_PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys)
{
assert(PyTuple_CheckExact(keys));
Py_ssize_t nkeys = PyTuple_GET_SIZE(keys);
@@ -403,7 +384,7 @@ fail:
// Extract a named attribute from the subject, with additional bookkeeping to
// raise TypeErrors for repeated lookups. On failure, return NULL (with no
// error set). Use _PyErr_Occurred(tstate) to disambiguate.
-static PyObject*
+static PyObject *
match_class_attr(PyThreadState *tstate, PyObject *subject, PyObject *type,
PyObject *name, PyObject *seen)
{
@@ -425,9 +406,9 @@ match_class_attr(PyThreadState *tstate, PyObject *subject, PyObject *type,
// On success (match), return a tuple of extracted attributes. On failure (no
// match), return NULL. Use _PyErr_Occurred(tstate) to disambiguate.
-static PyObject*
-match_class(PyThreadState *tstate, PyObject *subject, PyObject *type,
- Py_ssize_t nargs, PyObject *kwargs)
+PyObject*
+_PyEval_MatchClass(PyThreadState *tstate, PyObject *subject, PyObject *type,
+ Py_ssize_t nargs, PyObject *kwargs)
{
if (!PyType_Check(type)) {
const char *e = "called match pattern must be a class";
@@ -533,11 +514,6 @@ fail:
static int do_raise(PyThreadState *tstate, PyObject *exc, PyObject *cause);
-static int exception_group_match(
- PyObject* exc_value, PyObject *match_type,
- PyObject **match, PyObject **rest);
-
-static int unpack_iterable(PyThreadState *, PyObject *, int, int, PyObject **);
PyObject *
PyEval_EvalCode(PyObject *co, PyObject *globals, PyObject *locals)
@@ -827,7 +803,7 @@ resume_frame:
unbound_local_error:
{
- format_exc_check_arg(tstate, PyExc_UnboundLocalError,
+ _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError,
UNBOUNDLOCAL_ERROR_MSG,
PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)
);
@@ -1777,9 +1753,9 @@ raise_error:
complicated for inlining).
*/
-static int
-exception_group_match(PyObject* exc_value, PyObject *match_type,
- PyObject **match, PyObject **rest)
+int
+_PyEval_ExceptionGroupMatch(PyObject* exc_value, PyObject *match_type,
+ PyObject **match, PyObject **rest)
{
if (Py_IsNone(exc_value)) {
*match = Py_NewRef(Py_None);
@@ -1840,9 +1816,9 @@ exception_group_match(PyObject* exc_value, PyObject *match_type,
with a variable target.
*/
-static int
-unpack_iterable(PyThreadState *tstate, PyObject *v,
- int argcnt, int argcntafter, PyObject **sp)
+int
+_PyEval_UnpackIterable(PyThreadState *tstate, PyObject *v,
+ int argcnt, int argcntafter, PyObject **sp)
{
int i = 0, j = 0;
Py_ssize_t ll = 0;
@@ -2487,8 +2463,8 @@ import_from(PyThreadState *tstate, PyObject *v, PyObject *name)
#define CANNOT_EXCEPT_STAR_EG "catching ExceptionGroup with except* "\
"is not allowed. Use except instead."
-static int
-check_except_type_valid(PyThreadState *tstate, PyObject* right)
+int
+_PyEval_CheckExceptTypeValid(PyThreadState *tstate, PyObject* right)
{
if (PyTuple_Check(right)) {
Py_ssize_t i, length;
@@ -2512,10 +2488,10 @@ check_except_type_valid(PyThreadState *tstate, PyObject* right)
return 0;
}
-static int
-check_except_star_type_valid(PyThreadState *tstate, PyObject* right)
+int
+_PyEval_CheckExceptStarTypeValid(PyThreadState *tstate, PyObject* right)
{
- if (check_except_type_valid(tstate, right) < 0) {
+ if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) {
return -1;
}
@@ -2569,8 +2545,8 @@ check_args_iterable(PyThreadState *tstate, PyObject *func, PyObject *args)
return 0;
}
-static void
-format_kwargs_error(PyThreadState *tstate, PyObject *func, PyObject *kwargs)
+void
+_PyEval_FormatKwargsError(PyThreadState *tstate, PyObject *func, PyObject *kwargs)
{
/* _PyDict_MergeEx raises attribute
* error (percolated from an attempt
@@ -2611,9 +2587,9 @@ format_kwargs_error(PyThreadState *tstate, PyObject *func, PyObject *kwargs)
}
}
-static void
-format_exc_check_arg(PyThreadState *tstate, PyObject *exc,
- const char *format_str, PyObject *obj)
+void
+_PyEval_FormatExcCheckArg(PyThreadState *tstate, PyObject *exc,
+ const char *format_str, PyObject *obj)
{
const char *obj_str;
@@ -2640,8 +2616,8 @@ format_exc_check_arg(PyThreadState *tstate, PyObject *exc,
}
}
-static void
-format_exc_unbound(PyThreadState *tstate, PyCodeObject *co, int oparg)
+void
+_PyEval_FormatExcUnbound(PyThreadState *tstate, PyCodeObject *co, int oparg)
{
PyObject *name;
/* Don't stomp existing exception */
@@ -2649,16 +2625,16 @@ format_exc_unbound(PyThreadState *tstate, PyCodeObject *co, int oparg)
return;
name = PyTuple_GET_ITEM(co->co_localsplusnames, oparg);
if (oparg < PyCode_GetFirstFree(co)) {
- format_exc_check_arg(tstate, PyExc_UnboundLocalError,
- UNBOUNDLOCAL_ERROR_MSG, name);
+ _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError,
+ UNBOUNDLOCAL_ERROR_MSG, name);
} else {
- format_exc_check_arg(tstate, PyExc_NameError,
- UNBOUNDFREE_ERROR_MSG, name);
+ _PyEval_FormatExcCheckArg(tstate, PyExc_NameError,
+ UNBOUNDFREE_ERROR_MSG, name);
}
}
-static void
-format_awaitable_error(PyThreadState *tstate, PyTypeObject *type, int oparg)
+void
+_PyEval_FormatAwaitableError(PyThreadState *tstate, PyTypeObject *type, int oparg)
{
if (type->tp_as_async == NULL || type->tp_as_async->am_await == NULL) {
if (oparg == 1) {
@@ -2703,110 +2679,3 @@ void Py_LeaveRecursiveCall(void)
{
_Py_LeaveRecursiveCall();
}
-
-///////////////////// Experimental UOp Interpreter /////////////////////
-
-#undef ASSERT_KWNAMES_IS_NULL
-#define ASSERT_KWNAMES_IS_NULL() (void)0
-
-#undef DEOPT_IF
-#define DEOPT_IF(COND, INSTNAME) \
- if ((COND)) { \
- goto deoptimize; \
- }
-
-_PyInterpreterFrame *
-_PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject **stack_pointer)
-{
-#ifdef Py_DEBUG
- char *uop_debug = Py_GETENV("PYTHONUOPSDEBUG");
- int lltrace = 0;
- if (uop_debug != NULL && *uop_debug >= '0') {
- lltrace = *uop_debug - '0'; // TODO: Parse an int and all that
- }
-#define DPRINTF(level, ...) \
- if (lltrace >= (level)) { fprintf(stderr, __VA_ARGS__); }
-#else
-#define DPRINTF(level, ...)
-#endif
-
- DPRINTF(3,
- "Entering _PyUopExecute for %s (%s:%d) at byte offset %ld\n",
- PyUnicode_AsUTF8(_PyFrame_GetCode(frame)->co_qualname),
- PyUnicode_AsUTF8(_PyFrame_GetCode(frame)->co_filename),
- _PyFrame_GetCode(frame)->co_firstlineno,
- 2 * (long)(frame->prev_instr + 1 -
- (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive));
-
- PyThreadState *tstate = _PyThreadState_GET();
- _PyUOpExecutorObject *self = (_PyUOpExecutorObject *)executor;
-
- CHECK_EVAL_BREAKER();
-
- OBJECT_STAT_INC(optimization_traces_executed);
- _Py_CODEUNIT *ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive;
- int pc = 0;
- int opcode;
- int oparg;
- uint64_t operand;
-
- for (;;) {
- opcode = self->trace[pc].opcode;
- oparg = self->trace[pc].oparg;
- operand = self->trace[pc].operand;
- DPRINTF(3,
- "%4d: uop %s, oparg %d, operand %" PRIu64 ", stack_level %d\n",
- pc,
- opcode < 256 ? _PyOpcode_OpName[opcode] : _PyOpcode_uop_name[opcode],
- oparg,
- operand,
- (int)(stack_pointer - _PyFrame_Stackbase(frame)));
- pc++;
- OBJECT_STAT_INC(optimization_uops_executed);
- switch (opcode) {
-
-#undef ENABLE_SPECIALIZATION
-#define ENABLE_SPECIALIZATION 0
-#include "executor_cases.c.h"
-
- default:
- {
- fprintf(stderr, "Unknown uop %d, operand %" PRIu64 "\n", opcode, operand);
- Py_FatalError("Unknown uop");
- }
-
- }
- }
-
-unbound_local_error:
- format_exc_check_arg(tstate, PyExc_UnboundLocalError,
- UNBOUNDLOCAL_ERROR_MSG,
- PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)
- );
- goto error;
-
-pop_4_error:
- STACK_SHRINK(1);
-pop_3_error:
- STACK_SHRINK(1);
-pop_2_error:
- STACK_SHRINK(1);
-pop_1_error:
- STACK_SHRINK(1);
-error:
- // On ERROR_IF we return NULL as the frame.
- // The caller recovers the frame from cframe.current_frame.
- DPRINTF(2, "Error: [Opcode %d, operand %" PRIu64 "]\n", opcode, operand);
- _PyFrame_SetStackPointer(frame, stack_pointer);
- Py_DECREF(self);
- return NULL;
-
-deoptimize:
- // On DEOPT_IF we just repeat the last instruction.
- // This presumes nothing was popped from the stack (nor pushed).
- DPRINTF(2, "DEOPT: [Opcode %d, operand %" PRIu64 "]\n", opcode, operand);
- frame->prev_instr--; // Back up to just before destination
- _PyFrame_SetStackPointer(frame, stack_pointer);
- Py_DECREF(self);
- return frame;
-}
diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h
index c2c323317d1..8dc8b754485 100644
--- a/Python/ceval_macros.h
+++ b/Python/ceval_macros.h
@@ -1,4 +1,4 @@
-// Macros and other things needed by ceval.c and bytecodes.c
+// Macros and other things needed by ceval.c, executor.c, and bytecodes.c
/* Computed GOTOs, or
the-optimization-commonly-but-improperly-known-as-"threaded code"
@@ -304,6 +304,11 @@ GETITEM(PyObject *v, Py_ssize_t i) {
(COUNTER) += (1 << ADAPTIVE_BACKOFF_BITS); \
} while (0);
+#define UNBOUNDLOCAL_ERROR_MSG \
+ "cannot access local variable '%s' where it is not associated with a value"
+#define UNBOUNDFREE_ERROR_MSG \
+ "cannot access free variable '%s' where it is not associated with a value" \
+ " in enclosing scope"
#define NAME_ERROR_MSG "name '%.200s' is not defined"
#define KWNAMES_LEN() \
@@ -352,3 +357,10 @@ static const convertion_func_ptr CONVERSION_FUNCTIONS[4] = {
};
#define ASSERT_KWNAMES_IS_NULL() assert(kwnames == NULL)
+
+// GH-89279: Force inlining by using a macro.
+#if defined(_MSC_VER) && SIZEOF_INT == 4
+#define _Py_atomic_load_relaxed_int32(ATOMIC_VAL) (assert(sizeof((ATOMIC_VAL)->_value) == 4), *((volatile int*)&((ATOMIC_VAL)->_value)))
+#else
+#define _Py_atomic_load_relaxed_int32(ATOMIC_VAL) _Py_atomic_load_relaxed(ATOMIC_VAL)
+#endif
diff --git a/Python/executor.c b/Python/executor.c
new file mode 100644
index 00000000000..17322526a7b
--- /dev/null
+++ b/Python/executor.c
@@ -0,0 +1,125 @@
+#include "Python.h"
+
+#include "pycore_call.h"
+#include "pycore_ceval.h"
+#include "pycore_dict.h"
+#include "pycore_emscripten_signal.h"
+#include "pycore_intrinsics.h"
+#include "pycore_long.h"
+#include "pycore_object.h"
+#include "pycore_opcode_metadata.h"
+#include "pycore_opcode_utils.h"
+#include "pycore_pyerrors.h"
+#include "pycore_range.h"
+#include "pycore_sliceobject.h"
+#include "pycore_uops.h"
+
+#include "ceval_macros.h"
+
+
+#undef ASSERT_KWNAMES_IS_NULL
+#define ASSERT_KWNAMES_IS_NULL() (void)0
+
+#undef DEOPT_IF
+#define DEOPT_IF(COND, INSTNAME) \
+ if ((COND)) { \
+ goto deoptimize; \
+ }
+
+#undef ENABLE_SPECIALIZATION
+#define ENABLE_SPECIALIZATION 0
+
+
+_PyInterpreterFrame *
+_PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject **stack_pointer)
+{
+#ifdef Py_DEBUG
+ char *uop_debug = Py_GETENV("PYTHONUOPSDEBUG");
+ int lltrace = 0;
+ if (uop_debug != NULL && *uop_debug >= '0') {
+ lltrace = *uop_debug - '0'; // TODO: Parse an int and all that
+ }
+ #define DPRINTF(level, ...) \
+ if (lltrace >= (level)) { fprintf(stderr, __VA_ARGS__); }
+#else
+ #define DPRINTF(level, ...)
+#endif
+
+ DPRINTF(3,
+ "Entering _PyUopExecute for %s (%s:%d) at byte offset %ld\n",
+ PyUnicode_AsUTF8(_PyFrame_GetCode(frame)->co_qualname),
+ PyUnicode_AsUTF8(_PyFrame_GetCode(frame)->co_filename),
+ _PyFrame_GetCode(frame)->co_firstlineno,
+ 2 * (long)(frame->prev_instr + 1 -
+ (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive));
+
+ PyThreadState *tstate = _PyThreadState_GET();
+ _PyUOpExecutorObject *self = (_PyUOpExecutorObject *)executor;
+
+ CHECK_EVAL_BREAKER();
+
+ OBJECT_STAT_INC(optimization_traces_executed);
+ _Py_CODEUNIT *ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive;
+ int pc = 0;
+ int opcode;
+ int oparg;
+ uint64_t operand;
+
+ for (;;) {
+ opcode = self->trace[pc].opcode;
+ oparg = self->trace[pc].oparg;
+ operand = self->trace[pc].operand;
+ DPRINTF(3,
+ "%4d: uop %s, oparg %d, operand %" PRIu64 ", stack_level %d\n",
+ pc,
+ opcode < 256 ? _PyOpcode_OpName[opcode] : _PyOpcode_uop_name[opcode],
+ oparg,
+ operand,
+ (int)(stack_pointer - _PyFrame_Stackbase(frame)));
+ pc++;
+ OBJECT_STAT_INC(optimization_uops_executed);
+ switch (opcode) {
+
+#include "executor_cases.c.h"
+
+ default:
+ {
+ fprintf(stderr, "Unknown uop %d, operand %" PRIu64 "\n", opcode, operand);
+ Py_FatalError("Unknown uop");
+ }
+
+ }
+ }
+
+unbound_local_error:
+ _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError,
+ UNBOUNDLOCAL_ERROR_MSG,
+ PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)
+ );
+ goto error;
+
+pop_4_error:
+ STACK_SHRINK(1);
+pop_3_error:
+ STACK_SHRINK(1);
+pop_2_error:
+ STACK_SHRINK(1);
+pop_1_error:
+ STACK_SHRINK(1);
+error:
+ // On ERROR_IF we return NULL as the frame.
+ // The caller recovers the frame from cframe.current_frame.
+ DPRINTF(2, "Error: [Opcode %d, operand %" PRIu64 "]\n", opcode, operand);
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ Py_DECREF(self);
+ return NULL;
+
+deoptimize:
+ // On DEOPT_IF we just repeat the last instruction.
+ // This presumes nothing was popped from the stack (nor pushed).
+ DPRINTF(2, "DEOPT: [Opcode %d, operand %" PRIu64 "]\n", opcode, operand);
+ frame->prev_instr--; // Back up to just before destination
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ Py_DECREF(self);
+ return frame;
+}
diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h
index 5b38c6a5fd8..064965a1a1a 100644
--- a/Python/executor_cases.c.h
+++ b/Python/executor_cases.c.h
@@ -672,7 +672,7 @@
iter = _PyCoro_GetAwaitableIter(iterable);
if (iter == NULL) {
- format_awaitable_error(tstate, Py_TYPE(iterable), oparg);
+ _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg);
}
Py_DECREF(iterable);
@@ -758,9 +758,9 @@
err = PyObject_DelItem(ns, name);
// Can't use ERROR_IF here.
if (err != 0) {
- format_exc_check_arg(tstate, PyExc_NameError,
- NAME_ERROR_MSG,
- name);
+ _PyEval_FormatExcCheckArg(tstate, PyExc_NameError,
+ NAME_ERROR_MSG,
+ name);
goto error;
}
break;
@@ -780,7 +780,7 @@
DECREMENT_ADAPTIVE_COUNTER(cache->counter);
#endif /* ENABLE_SPECIALIZATION */
PyObject **top = stack_pointer + oparg - 1;
- int res = unpack_iterable(tstate, seq, oparg, -1, top);
+ int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top);
Py_DECREF(seq);
if (res == 0) goto pop_1_error;
STACK_SHRINK(1);
@@ -839,7 +839,7 @@
PyObject *seq = stack_pointer[-1];
int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8);
PyObject **top = stack_pointer + totalargs - 1;
- int res = unpack_iterable(tstate, seq, oparg & 0xFF, oparg >> 8, top);
+ int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top);
Py_DECREF(seq);
if (res == 0) goto pop_1_error;
STACK_GROW((oparg & 0xFF) + (oparg >> 8));
@@ -897,8 +897,8 @@
// Can't use ERROR_IF here.
if (err != 0) {
if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) {
- format_exc_check_arg(tstate, PyExc_NameError,
- NAME_ERROR_MSG, name);
+ _PyEval_FormatExcCheckArg(tstate, PyExc_NameError,
+ NAME_ERROR_MSG, name);
}
goto error;
}
@@ -941,7 +941,7 @@
goto error;
}
if (v == NULL) {
- format_exc_check_arg(
+ _PyEval_FormatExcCheckArg(
tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
goto error;
@@ -978,8 +978,8 @@
if (!_PyErr_Occurred(tstate)) {
/* _PyDict_LoadGlobal() returns NULL without raising
* an exception if the key doesn't exist */
- format_exc_check_arg(tstate, PyExc_NameError,
- NAME_ERROR_MSG, name);
+ _PyEval_FormatExcCheckArg(tstate, PyExc_NameError,
+ NAME_ERROR_MSG, name);
}
if (true) goto error;
}
@@ -994,7 +994,7 @@
/* namespace 2: builtins */
if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) goto error;
if (v == NULL) {
- format_exc_check_arg(
+ _PyEval_FormatExcCheckArg(
tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
if (true) goto error;
@@ -1080,7 +1080,7 @@
// Can't use ERROR_IF here.
// Fortunately we don't need its superpower.
if (oldobj == NULL) {
- format_exc_unbound(tstate, _PyFrame_GetCode(frame), oparg);
+ _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
goto error;
}
PyCell_SET(cell, NULL);
@@ -1104,7 +1104,7 @@
PyObject *cell = GETLOCAL(oparg);
value = PyCell_GET(cell);
if (value == NULL) {
- format_exc_unbound(tstate, _PyFrame_GetCode(frame), oparg);
+ _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
goto error;
}
Py_INCREF(value);
@@ -1118,7 +1118,7 @@
PyObject *cell = GETLOCAL(oparg);
value = PyCell_GET(cell);
if (value == NULL) {
- format_exc_unbound(tstate, _PyFrame_GetCode(frame), oparg);
+ _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
if (true) goto error;
}
Py_INCREF(value);
@@ -1348,7 +1348,7 @@
PyObject *dict = PEEK(oparg + 1); // update is still on the stack
if (_PyDict_MergeEx(dict, update, 2) < 0) {
- format_kwargs_error(tstate, PEEK(3 + oparg), update);
+ _PyEval_FormatKwargsError(tstate, PEEK(3 + oparg), update);
Py_DECREF(update);
if (true) goto pop_1_error;
}
@@ -1646,7 +1646,7 @@
PyObject *exc_value = stack_pointer[-2];
PyObject *rest;
PyObject *match;
- if (check_except_star_type_valid(tstate, match_type) < 0) {
+ if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) {
Py_DECREF(exc_value);
Py_DECREF(match_type);
if (true) goto pop_2_error;
@@ -1654,8 +1654,8 @@
match = NULL;
rest = NULL;
- int res = exception_group_match(exc_value, match_type,
- &match, &rest);
+ int res = _PyEval_ExceptionGroupMatch(exc_value, match_type,
+ &match, &rest);
Py_DECREF(exc_value);
Py_DECREF(match_type);
if (res < 0) goto pop_2_error;
@@ -1676,7 +1676,7 @@
PyObject *left = stack_pointer[-2];
PyObject *b;
assert(PyExceptionInstance_Check(left));
- if (check_except_type_valid(tstate, right) < 0) {
+ if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) {
Py_DECREF(right);
if (true) goto pop_1_error;
}
@@ -1723,7 +1723,7 @@
// Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or
// None on failure.
assert(PyTuple_CheckExact(names));
- attrs = match_class(tstate, subject, type, oparg, names);
+ attrs = _PyEval_MatchClass(tstate, subject, type, oparg, names);
Py_DECREF(subject);
Py_DECREF(type);
Py_DECREF(names);
@@ -1764,7 +1764,7 @@
PyObject *subject = stack_pointer[-2];
PyObject *values_or_none;
// On successful match, PUSH(values). Otherwise, PUSH(None).
- values_or_none = match_keys(tstate, subject, keys);
+ values_or_none = _PyEval_MatchKeys(tstate, subject, keys);
if (values_or_none == NULL) goto error;
STACK_GROW(1);
stack_pointer[-1] = values_or_none;
@@ -2462,10 +2462,10 @@
STAT_INC(BINARY_OP, deferred);
DECREMENT_ADAPTIVE_COUNTER(cache->counter);
#endif /* ENABLE_SPECIALIZATION */
- assert(0 <= oparg);
- assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops));
- assert(binary_ops[oparg]);
- res = binary_ops[oparg](lhs, rhs);
+ assert(NB_ADD <= oparg);
+ assert(oparg <= NB_INPLACE_XOR);
+ assert(_PyEval_BinaryOps[oparg]);
+ res = _PyEval_BinaryOps[oparg](lhs, rhs);
Py_DECREF(lhs);
Py_DECREF(rhs);
if (res == NULL) goto pop_2_error;
diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h
index 2d70966aa09..9d25fc60431 100644
--- a/Python/generated_cases.c.h
+++ b/Python/generated_cases.c.h
@@ -1068,7 +1068,7 @@
iter = _PyCoro_GetAwaitableIter(iterable);
if (iter == NULL) {
- format_awaitable_error(tstate, Py_TYPE(iterable), oparg);
+ _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg);
}
Py_DECREF(iterable);
@@ -1336,9 +1336,9 @@
err = PyObject_DelItem(ns, name);
// Can't use ERROR_IF here.
if (err != 0) {
- format_exc_check_arg(tstate, PyExc_NameError,
- NAME_ERROR_MSG,
- name);
+ _PyEval_FormatExcCheckArg(tstate, PyExc_NameError,
+ NAME_ERROR_MSG,
+ name);
goto error;
}
DISPATCH();
@@ -1359,7 +1359,7 @@
DECREMENT_ADAPTIVE_COUNTER(cache->counter);
#endif /* ENABLE_SPECIALIZATION */
PyObject **top = stack_pointer + oparg - 1;
- int res = unpack_iterable(tstate, seq, oparg, -1, top);
+ int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top);
Py_DECREF(seq);
if (res == 0) goto pop_1_error;
STACK_SHRINK(1);
@@ -1422,7 +1422,7 @@
PyObject *seq = stack_pointer[-1];
int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8);
PyObject **top = stack_pointer + totalargs - 1;
- int res = unpack_iterable(tstate, seq, oparg & 0xFF, oparg >> 8, top);
+ int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top);
Py_DECREF(seq);
if (res == 0) goto pop_1_error;
STACK_GROW((oparg & 0xFF) + (oparg >> 8));
@@ -1482,8 +1482,8 @@
// Can't use ERROR_IF here.
if (err != 0) {
if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) {
- format_exc_check_arg(tstate, PyExc_NameError,
- NAME_ERROR_MSG, name);
+ _PyEval_FormatExcCheckArg(tstate, PyExc_NameError,
+ NAME_ERROR_MSG, name);
}
goto error;
}
@@ -1543,7 +1543,7 @@
goto error;
}
if (v == NULL) {
- format_exc_check_arg(
+ _PyEval_FormatExcCheckArg(
tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
goto error;
@@ -1581,7 +1581,7 @@
goto error;
}
if (v == NULL) {
- format_exc_check_arg(
+ _PyEval_FormatExcCheckArg(
tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
goto error;
@@ -1621,8 +1621,8 @@
if (!_PyErr_Occurred(tstate)) {
/* _PyDict_LoadGlobal() returns NULL without raising
* an exception if the key doesn't exist */
- format_exc_check_arg(tstate, PyExc_NameError,
- NAME_ERROR_MSG, name);
+ _PyEval_FormatExcCheckArg(tstate, PyExc_NameError,
+ NAME_ERROR_MSG, name);
}
if (true) goto error;
}
@@ -1637,7 +1637,7 @@
/* namespace 2: builtins */
if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) goto error;
if (v == NULL) {
- format_exc_check_arg(
+ _PyEval_FormatExcCheckArg(
tstate, PyExc_NameError,
NAME_ERROR_MSG, name);
if (true) goto error;
@@ -1755,7 +1755,7 @@
// Can't use ERROR_IF here.
// Fortunately we don't need its superpower.
if (oldobj == NULL) {
- format_exc_unbound(tstate, _PyFrame_GetCode(frame), oparg);
+ _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
goto error;
}
PyCell_SET(cell, NULL);
@@ -1779,7 +1779,7 @@
PyObject *cell = GETLOCAL(oparg);
value = PyCell_GET(cell);
if (value == NULL) {
- format_exc_unbound(tstate, _PyFrame_GetCode(frame), oparg);
+ _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
goto error;
}
Py_INCREF(value);
@@ -1793,7 +1793,7 @@
PyObject *cell = GETLOCAL(oparg);
value = PyCell_GET(cell);
if (value == NULL) {
- format_exc_unbound(tstate, _PyFrame_GetCode(frame), oparg);
+ _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg);
if (true) goto error;
}
Py_INCREF(value);
@@ -2023,7 +2023,7 @@
PyObject *dict = PEEK(oparg + 1); // update is still on the stack
if (_PyDict_MergeEx(dict, update, 2) < 0) {
- format_kwargs_error(tstate, PEEK(3 + oparg), update);
+ _PyEval_FormatKwargsError(tstate, PEEK(3 + oparg), update);
Py_DECREF(update);
if (true) goto pop_1_error;
}
@@ -2679,7 +2679,7 @@
PyObject *exc_value = stack_pointer[-2];
PyObject *rest;
PyObject *match;
- if (check_except_star_type_valid(tstate, match_type) < 0) {
+ if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) {
Py_DECREF(exc_value);
Py_DECREF(match_type);
if (true) goto pop_2_error;
@@ -2687,8 +2687,8 @@
match = NULL;
rest = NULL;
- int res = exception_group_match(exc_value, match_type,
- &match, &rest);
+ int res = _PyEval_ExceptionGroupMatch(exc_value, match_type,
+ &match, &rest);
Py_DECREF(exc_value);
Py_DECREF(match_type);
if (res < 0) goto pop_2_error;
@@ -2709,7 +2709,7 @@
PyObject *left = stack_pointer[-2];
PyObject *b;
assert(PyExceptionInstance_Check(left));
- if (check_except_type_valid(tstate, right) < 0) {
+ if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) {
Py_DECREF(right);
if (true) goto pop_1_error;
}
@@ -2890,7 +2890,7 @@
// Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or
// None on failure.
assert(PyTuple_CheckExact(names));
- attrs = match_class(tstate, subject, type, oparg, names);
+ attrs = _PyEval_MatchClass(tstate, subject, type, oparg, names);
Py_DECREF(subject);
Py_DECREF(type);
Py_DECREF(names);
@@ -2931,7 +2931,7 @@
PyObject *subject = stack_pointer[-2];
PyObject *values_or_none;
// On successful match, PUSH(values). Otherwise, PUSH(None).
- values_or_none = match_keys(tstate, subject, keys);
+ values_or_none = _PyEval_MatchKeys(tstate, subject, keys);
if (values_or_none == NULL) goto error;
STACK_GROW(1);
stack_pointer[-1] = values_or_none;
@@ -4464,10 +4464,10 @@
STAT_INC(BINARY_OP, deferred);
DECREMENT_ADAPTIVE_COUNTER(cache->counter);
#endif /* ENABLE_SPECIALIZATION */
- assert(0 <= oparg);
- assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops));
- assert(binary_ops[oparg]);
- res = binary_ops[oparg](lhs, rhs);
+ assert(NB_ADD <= oparg);
+ assert(oparg <= NB_INPLACE_XOR);
+ assert(_PyEval_BinaryOps[oparg]);
+ res = _PyEval_BinaryOps[oparg](lhs, rhs);
Py_DECREF(lhs);
Py_DECREF(rhs);
if (res == NULL) goto pop_2_error;
diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv
index 228ca8648a2..cf6d3b24435 100644
--- a/Tools/c-analyzer/cpython/ignored.tsv
+++ b/Tools/c-analyzer/cpython/ignored.tsv
@@ -326,7 +326,7 @@ Parser/parser.c - reserved_keywords -
Parser/parser.c - soft_keywords -
Parser/tokenizer.c - type_comment_prefix -
Python/ast_opt.c fold_unaryop ops -
-Python/ceval.c - binary_ops -
+Python/ceval.c - _PyEval_BinaryOps -
Python/ceval.c - _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS -
Python/codecs.c - Py_hexdigits -
Python/codecs.c - ucnhash_capi -