Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

gh-116968: Reimplement Tier 2 counters (#117144) 

Introduce a unified 16-bit backoff counter type (``_Py_BackoffCounter``),
shared between the Tier 1 adaptive specializer and the Tier 2 optimizer. The
API used for adaptive specialization counters is changed but the behavior is
(supposed to be) identical.

The behavior of the Tier 2 counters is changed:
- There are no longer dynamic thresholds (we never varied these).
- All counters now use the same exponential backoff.
- The counter for ``JUMP_BACKWARD`` starts counting down from 16.
- The ``temperature`` in side exits starts counting down from 64.
This commit is contained in:
Guido van Rossum 2024-04-04 08:03:27 -07:00 committed by GitHub
parent 63bbe77d9b
commit 060a96f1a9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
19 changed files with 313 additions and 235 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
Include/cpython/code.h
View File

@ -24,6 +24,16 @@ typedef struct _Py_GlobalMonitors {
uint8_t tools[_PY_MONITORING_UNGROUPED_EVENTS]; uint8_t tools[_PY_MONITORING_UNGROUPED_EVENTS];
} _Py_GlobalMonitors; } _Py_GlobalMonitors;
typedef struct {
union {
struct {
uint16_t backoff : 4;
uint16_t value : 12;
};
uint16_t as_counter; // For printf("%#x", ...)
};
} _Py_BackoffCounter;
/* Each instruction in a code object is a fixed-width value, /* Each instruction in a code object is a fixed-width value,
* currently 2 bytes: 1-byte opcode + 1-byte oparg. The EXTENDED_ARG * currently 2 bytes: 1-byte opcode + 1-byte oparg. The EXTENDED_ARG
* opcode allows for larger values but the current limit is 3 uses * opcode allows for larger values but the current limit is 3 uses
@ -39,6 +49,7 @@ typedef union {
uint8_t code; uint8_t code;
uint8_t arg; uint8_t arg;
} op; } op;
_Py_BackoffCounter counter; // First cache entry of specializable op
} _Py_CODEUNIT; } _Py_CODEUNIT;

15
Include/cpython/optimizer.h
View File

@ -89,7 +89,7 @@ static inline uint16_t uop_get_error_target(const _PyUOpInstruction *inst)
typedef struct _exit_data { typedef struct _exit_data {
uint32_t target; uint32_t target;
int16_t temperature; _Py_BackoffCounter temperature;
const struct _PyExecutorObject *executor; const struct _PyExecutorObject *executor;
} _PyExitData; } _PyExitData;
@ -115,11 +115,6 @@ typedef int (*optimize_func)(
struct _PyOptimizerObject { struct _PyOptimizerObject {
PyObject_HEAD PyObject_HEAD
optimize_func optimize; optimize_func optimize;
/* These thresholds are treated as signed so do not exceed INT16_MAX
* Use INT16_MAX to indicate that the optimizer should never be called */
uint16_t resume_threshold;
uint16_t side_threshold;
uint16_t backedge_threshold;
/* Data needed by the optimizer goes here, but is opaque to the VM */ /* Data needed by the optimizer goes here, but is opaque to the VM */
}; };
@ -151,14 +146,6 @@ extern void _Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_inval
PyAPI_FUNC(PyObject *)PyUnstable_Optimizer_NewCounter(void); PyAPI_FUNC(PyObject *)PyUnstable_Optimizer_NewCounter(void);
PyAPI_FUNC(PyObject *)PyUnstable_Optimizer_NewUOpOptimizer(void); PyAPI_FUNC(PyObject *)PyUnstable_Optimizer_NewUOpOptimizer(void);
#define OPTIMIZER_BITS_IN_COUNTER 4
/* Minimum of 16 additional executions before retry */
#define MIN_TIER2_BACKOFF 4
#define MAX_TIER2_BACKOFF (15 - OPTIMIZER_BITS_IN_COUNTER)
#define OPTIMIZER_BITS_MASK ((1 << OPTIMIZER_BITS_IN_COUNTER) - 1)
/* A value <= UINT16_MAX but large enough that when shifted is > UINT16_MAX */
#define OPTIMIZER_UNREACHABLE_THRESHOLD UINT16_MAX
#define _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS 3 #define _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS 3
#define _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS 6 #define _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS 6

128
Include/internal/pycore_backoff.h Normal file
View File

@ -0,0 +1,128 @@
#ifndef Py_INTERNAL_BACKOFF_H
#define Py_INTERNAL_BACKOFF_H
#ifdef __cplusplus
extern "C" {
#endif
#ifndef Py_BUILD_CORE
# error "this header requires Py_BUILD_CORE define"
#endif
#include <assert.h>
#include <stdbool.h>
#include <stdint.h>
/* 16-bit countdown counters using exponential backoff.
These are used by the adaptive specializer to count down until
it is time to specialize an instruction. If specialization fails
the counter is reset using exponential backoff.
Another use is for the Tier 2 optimizer to decide when to create
a new Tier 2 trace (executor). Again, exponential backoff is used.
The 16-bit counter is structured as a 12-bit unsigned 'value'
and a 4-bit 'backoff' field. When resetting the counter, the
backoff field is incremented (until it reaches a limit) and the
value is set to a bit mask representing the value 2**backoff - 1.
The maximum backoff is 12 (the number of value bits).
There is an exceptional value which must not be updated, 0xFFFF.
*/
#define UNREACHABLE_BACKOFF 0xFFFF
static inline bool
is_unreachable_backoff_counter(_Py_BackoffCounter counter)
{
return counter.as_counter == UNREACHABLE_BACKOFF;
}
static inline _Py_BackoffCounter
make_backoff_counter(uint16_t value, uint16_t backoff)
{
assert(backoff <= 15);
assert(value <= 0xFFF);
return (_Py_BackoffCounter){.value = value, .backoff = backoff};
}
static inline _Py_BackoffCounter
forge_backoff_counter(uint16_t counter)
{
return (_Py_BackoffCounter){.as_counter = counter};
}
static inline _Py_BackoffCounter
restart_backoff_counter(_Py_BackoffCounter counter)
{
assert(!is_unreachable_backoff_counter(counter));
if (counter.backoff < 12) {
return make_backoff_counter((1 << (counter.backoff + 1)) - 1, counter.backoff + 1);
}
else {
return make_backoff_counter((1 << 12) - 1, 12);
}
}
static inline _Py_BackoffCounter
pause_backoff_counter(_Py_BackoffCounter counter)
{
return make_backoff_counter(counter.value | 1, counter.backoff);
}
static inline _Py_BackoffCounter
advance_backoff_counter(_Py_BackoffCounter counter)
{
if (!is_unreachable_backoff_counter(counter)) {
return make_backoff_counter((counter.value - 1) & 0xFFF, counter.backoff);
}
else {
return counter;
}
}
static inline bool
backoff_counter_triggers(_Py_BackoffCounter counter)
{
return counter.value == 0;
}
/* Initial JUMP_BACKWARD counter.
* This determines when we create a trace for a loop.
* Backoff sequence 16, 32, 64, 128, 256, 512, 1024, 2048, 4096. */
#define JUMP_BACKWARD_INITIAL_VALUE 16
#define JUMP_BACKWARD_INITIAL_BACKOFF 4
static inline _Py_BackoffCounter
initial_jump_backoff_counter(void)
{
return make_backoff_counter(JUMP_BACKWARD_INITIAL_VALUE,
JUMP_BACKWARD_INITIAL_BACKOFF);
}
/* Initial exit temperature.
* Must be larger than ADAPTIVE_COOLDOWN_VALUE,
* otherwise when a side exit warms up we may construct
* a new trace before the Tier 1 code has properly re-specialized.
* Backoff sequence 64, 128, 256, 512, 1024, 2048, 4096. */
#define COLD_EXIT_INITIAL_VALUE 64
#define COLD_EXIT_INITIAL_BACKOFF 6
static inline _Py_BackoffCounter
initial_temperature_backoff_counter(void)
{
return make_backoff_counter(COLD_EXIT_INITIAL_VALUE,
COLD_EXIT_INITIAL_BACKOFF);
}
/* Unreachable backoff counter. */
static inline _Py_BackoffCounter
initial_unreachable_backoff_counter(void)
{
return forge_backoff_counter(UNREACHABLE_BACKOFF);
}
#ifdef __cplusplus
}
#endif
#endif /* !Py_INTERNAL_BACKOFF_H */

64
Include/internal/pycore_code.h
View File

@ -31,7 +31,7 @@ extern "C" {
#define CACHE_ENTRIES(cache) (sizeof(cache)/sizeof(_Py_CODEUNIT)) #define CACHE_ENTRIES(cache) (sizeof(cache)/sizeof(_Py_CODEUNIT))
typedef struct { typedef struct {
uint16_t counter; _Py_BackoffCounter counter;
uint16_t module_keys_version; uint16_t module_keys_version;
uint16_t builtin_keys_version; uint16_t builtin_keys_version;
uint16_t index; uint16_t index;
@ -40,44 +40,44 @@ typedef struct {
#define INLINE_CACHE_ENTRIES_LOAD_GLOBAL CACHE_ENTRIES(_PyLoadGlobalCache) #define INLINE_CACHE_ENTRIES_LOAD_GLOBAL CACHE_ENTRIES(_PyLoadGlobalCache)
typedef struct { typedef struct {
uint16_t counter; _Py_BackoffCounter counter;
} _PyBinaryOpCache; } _PyBinaryOpCache;
#define INLINE_CACHE_ENTRIES_BINARY_OP CACHE_ENTRIES(_PyBinaryOpCache) #define INLINE_CACHE_ENTRIES_BINARY_OP CACHE_ENTRIES(_PyBinaryOpCache)
typedef struct { typedef struct {
uint16_t counter; _Py_BackoffCounter counter;
} _PyUnpackSequenceCache; } _PyUnpackSequenceCache;
#define INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE \ #define INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE \
CACHE_ENTRIES(_PyUnpackSequenceCache) CACHE_ENTRIES(_PyUnpackSequenceCache)
typedef struct { typedef struct {
uint16_t counter; _Py_BackoffCounter counter;
} _PyCompareOpCache; } _PyCompareOpCache;
#define INLINE_CACHE_ENTRIES_COMPARE_OP CACHE_ENTRIES(_PyCompareOpCache) #define INLINE_CACHE_ENTRIES_COMPARE_OP CACHE_ENTRIES(_PyCompareOpCache)
typedef struct { typedef struct {
uint16_t counter; _Py_BackoffCounter counter;
} _PyBinarySubscrCache; } _PyBinarySubscrCache;
#define INLINE_CACHE_ENTRIES_BINARY_SUBSCR CACHE_ENTRIES(_PyBinarySubscrCache) #define INLINE_CACHE_ENTRIES_BINARY_SUBSCR CACHE_ENTRIES(_PyBinarySubscrCache)
typedef struct { typedef struct {
uint16_t counter; _Py_BackoffCounter counter;
} _PySuperAttrCache; } _PySuperAttrCache;
#define INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR CACHE_ENTRIES(_PySuperAttrCache) #define INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR CACHE_ENTRIES(_PySuperAttrCache)
typedef struct { typedef struct {
uint16_t counter; _Py_BackoffCounter counter;
uint16_t version[2]; uint16_t version[2];
uint16_t index; uint16_t index;
} _PyAttrCache; } _PyAttrCache;
typedef struct { typedef struct {
uint16_t counter; _Py_BackoffCounter counter;
uint16_t type_version[2]; uint16_t type_version[2];
union { union {
uint16_t keys_version[2]; uint16_t keys_version[2];
@ -93,39 +93,39 @@ typedef struct {
#define INLINE_CACHE_ENTRIES_STORE_ATTR CACHE_ENTRIES(_PyAttrCache) #define INLINE_CACHE_ENTRIES_STORE_ATTR CACHE_ENTRIES(_PyAttrCache)
typedef struct { typedef struct {
uint16_t counter; _Py_BackoffCounter counter;
uint16_t func_version[2]; uint16_t func_version[2];
} _PyCallCache; } _PyCallCache;
#define INLINE_CACHE_ENTRIES_CALL CACHE_ENTRIES(_PyCallCache) #define INLINE_CACHE_ENTRIES_CALL CACHE_ENTRIES(_PyCallCache)
typedef struct { typedef struct {
uint16_t counter; _Py_BackoffCounter counter;
} _PyStoreSubscrCache; } _PyStoreSubscrCache;
#define INLINE_CACHE_ENTRIES_STORE_SUBSCR CACHE_ENTRIES(_PyStoreSubscrCache) #define INLINE_CACHE_ENTRIES_STORE_SUBSCR CACHE_ENTRIES(_PyStoreSubscrCache)
typedef struct { typedef struct {
uint16_t counter; _Py_BackoffCounter counter;
} _PyForIterCache; } _PyForIterCache;
#define INLINE_CACHE_ENTRIES_FOR_ITER CACHE_ENTRIES(_PyForIterCache) #define INLINE_CACHE_ENTRIES_FOR_ITER CACHE_ENTRIES(_PyForIterCache)
typedef struct { typedef struct {
uint16_t counter; _Py_BackoffCounter counter;
} _PySendCache; } _PySendCache;
#define INLINE_CACHE_ENTRIES_SEND CACHE_ENTRIES(_PySendCache) #define INLINE_CACHE_ENTRIES_SEND CACHE_ENTRIES(_PySendCache)
typedef struct { typedef struct {
uint16_t counter; _Py_BackoffCounter counter;
uint16_t version[2]; uint16_t version[2];
} _PyToBoolCache; } _PyToBoolCache;
#define INLINE_CACHE_ENTRIES_TO_BOOL CACHE_ENTRIES(_PyToBoolCache) #define INLINE_CACHE_ENTRIES_TO_BOOL CACHE_ENTRIES(_PyToBoolCache)
typedef struct { typedef struct {
uint16_t counter; _Py_BackoffCounter counter;
} _PyContainsOpCache; } _PyContainsOpCache;
#define INLINE_CACHE_ENTRIES_CONTAINS_OP CACHE_ENTRIES(_PyContainsOpCache) #define INLINE_CACHE_ENTRIES_CONTAINS_OP CACHE_ENTRIES(_PyContainsOpCache)
@ -451,18 +451,14 @@ write_location_entry_start(uint8_t *ptr, int code, int length)
/** Counters /** Counters
* The first 16-bit value in each inline cache is a counter. * The first 16-bit value in each inline cache is a counter.
* When counting misses, the counter is treated as a simple unsigned value.
* *
* When counting executions until the next specialization attempt, * When counting executions until the next specialization attempt,
* exponential backoff is used to reduce the number of specialization failures. * exponential backoff is used to reduce the number of specialization failures.
* The high 12 bits store the counter, the low 4 bits store the backoff exponent. * See pycore_backoff.h for more details.
* On a specialization failure, the backoff exponent is incremented and the * On a specialization failure, the backoff counter is restarted.
* counter set to (2**backoff - 1).
* Backoff == 6 -> starting counter == 63, backoff == 10 -> starting counter == 1023.
*/ */
/* With a 16-bit counter, we have 12 bits for the counter value, and 4 bits for the backoff */ #include "pycore_backoff.h"
#define ADAPTIVE_BACKOFF_BITS 4
// A value of 1 means that we attempt to specialize the *second* time each // A value of 1 means that we attempt to specialize the *second* time each
// instruction is executed. Executing twice is a much better indicator of // instruction is executed. Executing twice is a much better indicator of
@ -480,36 +476,30 @@ write_location_entry_start(uint8_t *ptr, int code, int length)
#define ADAPTIVE_COOLDOWN_VALUE 52 #define ADAPTIVE_COOLDOWN_VALUE 52
#define ADAPTIVE_COOLDOWN_BACKOFF 0 #define ADAPTIVE_COOLDOWN_BACKOFF 0
#define MAX_BACKOFF_VALUE (16 - ADAPTIVE_BACKOFF_BITS) // Can't assert this in pycore_backoff.h because of header order dependencies
static_assert(COLD_EXIT_INITIAL_VALUE > ADAPTIVE_COOLDOWN_VALUE,
"Cold exit value should be larger than adaptive cooldown value");
static inline _Py_BackoffCounter
static inline uint16_t
adaptive_counter_bits(uint16_t value, uint16_t backoff) { adaptive_counter_bits(uint16_t value, uint16_t backoff) {
return ((value << ADAPTIVE_BACKOFF_BITS) return make_backoff_counter(value, backoff);
| (backoff & ((1 << ADAPTIVE_BACKOFF_BITS) - 1)));
} }
static inline uint16_t static inline _Py_BackoffCounter
adaptive_counter_warmup(void) { adaptive_counter_warmup(void) {
return adaptive_counter_bits(ADAPTIVE_WARMUP_VALUE, return adaptive_counter_bits(ADAPTIVE_WARMUP_VALUE,
ADAPTIVE_WARMUP_BACKOFF); ADAPTIVE_WARMUP_BACKOFF);
} }
static inline uint16_t static inline _Py_BackoffCounter
adaptive_counter_cooldown(void) { adaptive_counter_cooldown(void) {
return adaptive_counter_bits(ADAPTIVE_COOLDOWN_VALUE, return adaptive_counter_bits(ADAPTIVE_COOLDOWN_VALUE,
ADAPTIVE_COOLDOWN_BACKOFF); ADAPTIVE_COOLDOWN_BACKOFF);
} }
static inline uint16_t static inline _Py_BackoffCounter
adaptive_counter_backoff(uint16_t counter) { adaptive_counter_backoff(_Py_BackoffCounter counter) {
uint16_t backoff = counter & ((1 << ADAPTIVE_BACKOFF_BITS) - 1); return restart_backoff_counter(counter);
backoff++;
if (backoff > MAX_BACKOFF_VALUE) {
backoff = MAX_BACKOFF_VALUE;
}
uint16_t value = (uint16_t)(1 << backoff) - 1;
return adaptive_counter_bits(value, backoff);
} }

6
Include/internal/pycore_interp.h
View File

@ -239,12 +239,6 @@ struct _is {
_PyOptimizerObject *optimizer; _PyOptimizerObject *optimizer;
_PyExecutorObject *executor_list_head; _PyExecutorObject *executor_list_head;
/* These two values are shifted and offset to speed up check in JUMP_BACKWARD */
uint32_t optimizer_resume_threshold;
uint32_t optimizer_backedge_threshold;
uint16_t optimizer_side_threshold;
_rare_events rare_events; _rare_events rare_events;
PyDict_WatchCallback builtins_dict_watcher; PyDict_WatchCallback builtins_dict_watcher;

16
Lib/test/test_capi/test_opt.py
View File

@ -10,6 +10,7 @@ import _testinternalcapi
from test.support import script_helper, requires_specialization from test.support import script_helper, requires_specialization
from _testinternalcapi import TIER2_THRESHOLD
@contextlib.contextmanager @contextlib.contextmanager
def temporary_optimizer(opt): def temporary_optimizer(opt):
@ -69,7 +70,8 @@ class TestOptimizerAPI(unittest.TestCase):
self.assertEqual(opt.get_count(), 0) self.assertEqual(opt.get_count(), 0)
with clear_executors(loop): with clear_executors(loop):
loop() loop()
self.assertEqual(opt.get_count(), 1000) # Subtract because optimizer doesn't kick in sooner
self.assertEqual(opt.get_count(), 1000 - TIER2_THRESHOLD)
def test_long_loop(self): def test_long_loop(self):
"Check that we aren't confused by EXTENDED_ARG" "Check that we aren't confused by EXTENDED_ARG"
@ -81,7 +83,7 @@ class TestOptimizerAPI(unittest.TestCase):
pass pass
def long_loop(): def long_loop():
for _ in range(10): for _ in range(20):
nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop();
nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop();
nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop();
@ -96,7 +98,7 @@ class TestOptimizerAPI(unittest.TestCase):
with temporary_optimizer(opt): with temporary_optimizer(opt):
self.assertEqual(opt.get_count(), 0) self.assertEqual(opt.get_count(), 0)
long_loop() long_loop()
self.assertEqual(opt.get_count(), 10) self.assertEqual(opt.get_count(), 20 - TIER2_THRESHOLD) # Need iterations to warm up
def test_code_restore_for_ENTER_EXECUTOR(self): def test_code_restore_for_ENTER_EXECUTOR(self):
def testfunc(x): def testfunc(x):
@ -932,10 +934,10 @@ class TestUopsOptimization(unittest.TestCase):
exec(src, ns, ns) exec(src, ns, ns)
testfunc = ns['testfunc'] testfunc = ns['testfunc']
ns['_test_global'] = 0 ns['_test_global'] = 0
_, ex = self._run_with_optimizer(testfunc, 16) _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD)
self.assertIsNone(ex) self.assertIsNone(ex)
ns['_test_global'] = 1 ns['_test_global'] = 1
_, ex = self._run_with_optimizer(testfunc, 16) _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD)
self.assertIsNotNone(ex) self.assertIsNotNone(ex)
uops = get_opnames(ex) uops = get_opnames(ex)
self.assertNotIn("_GUARD_BOTH_INT", uops) self.assertNotIn("_GUARD_BOTH_INT", uops)
@ -946,10 +948,10 @@ class TestUopsOptimization(unittest.TestCase):
exec(src, ns, ns) exec(src, ns, ns)
testfunc = ns['testfunc'] testfunc = ns['testfunc']
ns['_test_global'] = 0 ns['_test_global'] = 0
_, ex = self._run_with_optimizer(testfunc, 16) _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD)
self.assertIsNone(ex) self.assertIsNone(ex)
ns['_test_global'] = 3.14 ns['_test_global'] = 3.14
_, ex = self._run_with_optimizer(testfunc, 16) _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD)
self.assertIsNone(ex) self.assertIsNone(ex)
def test_combine_stack_space_checks_sequential(self): def test_combine_stack_space_checks_sequential(self):

1
Makefile.pre.in
View File

@ -1124,6 +1124,7 @@ PYTHON_HEADERS= \
$(srcdir)/Include/internal/pycore_ast.h \ $(srcdir)/Include/internal/pycore_ast.h \
$(srcdir)/Include/internal/pycore_ast_state.h \ $(srcdir)/Include/internal/pycore_ast_state.h \
$(srcdir)/Include/internal/pycore_atexit.h \ $(srcdir)/Include/internal/pycore_atexit.h \
$(srcdir)/Include/internal/pycore_backoff.h \
$(srcdir)/Include/internal/pycore_bitutils.h \ $(srcdir)/Include/internal/pycore_bitutils.h \
$(srcdir)/Include/internal/pycore_blocks_output_buffer.h \ $(srcdir)/Include/internal/pycore_blocks_output_buffer.h \
$(srcdir)/Include/internal/pycore_brc.h \ $(srcdir)/Include/internal/pycore_brc.h \

11
Misc/NEWS.d/next/Core and Builtins/2024-04-03-13-44-04.gh-issue-116968.zgcdG2.rst Normal file
View File

@ -0,0 +1,11 @@
Introduce a unified 16-bit backoff counter type (``_Py_BackoffCounter``),
shared between the Tier 1 adaptive specializer and the Tier 2 optimizer. The
API used for adaptive specialization counters is changed but the behavior is
(supposed to be) identical.
The behavior of the Tier 2 counters is changed:
* There are no longer dynamic thresholds (we never varied these).
* All counters now use the same exponential backoff.
* The counter for ``JUMP_BACKWARD`` starts counting down from 16.
* The ``temperature`` in side exits starts counting down from 64.

6
Modules/_testinternalcapi.c
View File

@ -10,6 +10,7 @@
#undef NDEBUG #undef NDEBUG
#include "Python.h" #include "Python.h"
#include "pycore_backoff.h" // JUMP_BACKWARD_INITIAL_VALUE
#include "pycore_bitutils.h" // _Py_bswap32() #include "pycore_bitutils.h" // _Py_bswap32()
#include "pycore_bytesobject.h" // _PyBytes_Find() #include "pycore_bytesobject.h" // _PyBytes_Find()
#include "pycore_ceval.h" // _PyEval_AddPendingCall() #include "pycore_ceval.h" // _PyEval_AddPendingCall()
@ -1819,6 +1820,11 @@ module_exec(PyObject *module)
return 1; return 1;
} }
if (PyModule_Add(module, "TIER2_THRESHOLD",
PyLong_FromLong(JUMP_BACKWARD_INITIAL_VALUE)) < 0) {
return 1;
}
return 0; return 0;
} }

1
PCbuild/pythoncore.vcxproj
View File

@ -204,6 +204,7 @@
<ClInclude Include="..\Include\internal\pycore_ast.h" /> <ClInclude Include="..\Include\internal\pycore_ast.h" />
<ClInclude Include="..\Include\internal\pycore_ast_state.h" /> <ClInclude Include="..\Include\internal\pycore_ast_state.h" />
<ClInclude Include="..\Include\internal\pycore_atexit.h" /> <ClInclude Include="..\Include\internal\pycore_atexit.h" />
<ClInclude Include="..\Include\internal\pycore_backoff.h" />
<ClInclude Include="..\Include\internal\pycore_bitutils.h" /> <ClInclude Include="..\Include\internal\pycore_bitutils.h" />
<ClInclude Include="..\Include\internal\pycore_brc.h" /> <ClInclude Include="..\Include\internal\pycore_brc.h" />
<ClInclude Include="..\Include\internal\pycore_bytes_methods.h" /> <ClInclude Include="..\Include\internal\pycore_bytes_methods.h" />

97
Python/bytecodes.c
View File

@ -8,6 +8,7 @@
#include "Python.h" #include "Python.h"
#include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_abstract.h" // _PyIndex_Check()
#include "pycore_backoff.h"
#include "pycore_cell.h" // PyCell_GetRef() #include "pycore_cell.h" // PyCell_GetRef()
#include "pycore_code.h" #include "pycore_code.h"
#include "pycore_emscripten_signal.h" // _Py_CHECK_EMSCRIPTEN_SIGNALS #include "pycore_emscripten_signal.h" // _Py_CHECK_EMSCRIPTEN_SIGNALS
@ -326,13 +327,13 @@ dummy_func(
specializing op(_SPECIALIZE_TO_BOOL, (counter/1, value -- value)) { specializing op(_SPECIALIZE_TO_BOOL, (counter/1, value -- value)) {
#if ENABLE_SPECIALIZATION #if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr; next_instr = this_instr;
_Py_Specialize_ToBool(value, next_instr); _Py_Specialize_ToBool(value, next_instr);
DISPATCH_SAME_OPARG(); DISPATCH_SAME_OPARG();
} }
STAT_INC(TO_BOOL, deferred); STAT_INC(TO_BOOL, deferred);
DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */ #endif /* ENABLE_SPECIALIZATION */
} }
@ -551,13 +552,13 @@ dummy_func(
specializing op(_SPECIALIZE_BINARY_SUBSCR, (counter/1, container, sub -- container, sub)) { specializing op(_SPECIALIZE_BINARY_SUBSCR, (counter/1, container, sub -- container, sub)) {
#if ENABLE_SPECIALIZATION #if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr; next_instr = this_instr;
_Py_Specialize_BinarySubscr(container, sub, next_instr); _Py_Specialize_BinarySubscr(container, sub, next_instr);
DISPATCH_SAME_OPARG(); DISPATCH_SAME_OPARG();
} }
STAT_INC(BINARY_SUBSCR, deferred); STAT_INC(BINARY_SUBSCR, deferred);
DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */ #endif /* ENABLE_SPECIALIZATION */
} }
@ -698,13 +699,13 @@ dummy_func(
specializing op(_SPECIALIZE_STORE_SUBSCR, (counter/1, container, sub -- container, sub)) { specializing op(_SPECIALIZE_STORE_SUBSCR, (counter/1, container, sub -- container, sub)) {
#if ENABLE_SPECIALIZATION #if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr; next_instr = this_instr;
_Py_Specialize_StoreSubscr(container, sub, next_instr); _Py_Specialize_StoreSubscr(container, sub, next_instr);
DISPATCH_SAME_OPARG(); DISPATCH_SAME_OPARG();
} }
STAT_INC(STORE_SUBSCR, deferred); STAT_INC(STORE_SUBSCR, deferred);
DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */ #endif /* ENABLE_SPECIALIZATION */
} }
@ -982,13 +983,13 @@ dummy_func(
specializing op(_SPECIALIZE_SEND, (counter/1, receiver, unused -- receiver, unused)) { specializing op(_SPECIALIZE_SEND, (counter/1, receiver, unused -- receiver, unused)) {
#if ENABLE_SPECIALIZATION #if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr; next_instr = this_instr;
_Py_Specialize_Send(receiver, next_instr); _Py_Specialize_Send(receiver, next_instr);
DISPATCH_SAME_OPARG(); DISPATCH_SAME_OPARG();
} }
STAT_INC(SEND, deferred); STAT_INC(SEND, deferred);
DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */ #endif /* ENABLE_SPECIALIZATION */
} }
@ -1211,13 +1212,13 @@ dummy_func(
specializing op(_SPECIALIZE_UNPACK_SEQUENCE, (counter/1, seq -- seq)) { specializing op(_SPECIALIZE_UNPACK_SEQUENCE, (counter/1, seq -- seq)) {
#if ENABLE_SPECIALIZATION #if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr; next_instr = this_instr;
_Py_Specialize_UnpackSequence(seq, next_instr, oparg); _Py_Specialize_UnpackSequence(seq, next_instr, oparg);
DISPATCH_SAME_OPARG(); DISPATCH_SAME_OPARG();
} }
STAT_INC(UNPACK_SEQUENCE, deferred); STAT_INC(UNPACK_SEQUENCE, deferred);
DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */ #endif /* ENABLE_SPECIALIZATION */
(void)seq; (void)seq;
(void)counter; (void)counter;
@ -1280,14 +1281,14 @@ dummy_func(
specializing op(_SPECIALIZE_STORE_ATTR, (counter/1, owner -- owner)) { specializing op(_SPECIALIZE_STORE_ATTR, (counter/1, owner -- owner)) {
#if ENABLE_SPECIALIZATION #if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
next_instr = this_instr; next_instr = this_instr;
_Py_Specialize_StoreAttr(owner, next_instr, name); _Py_Specialize_StoreAttr(owner, next_instr, name);
DISPATCH_SAME_OPARG(); DISPATCH_SAME_OPARG();
} }
STAT_INC(STORE_ATTR, deferred); STAT_INC(STORE_ATTR, deferred);
DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */ #endif /* ENABLE_SPECIALIZATION */
} }
@ -1398,14 +1399,14 @@ dummy_func(
specializing op(_SPECIALIZE_LOAD_GLOBAL, (counter/1 -- )) { specializing op(_SPECIALIZE_LOAD_GLOBAL, (counter/1 -- )) {
#if ENABLE_SPECIALIZATION #if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
next_instr = this_instr; next_instr = this_instr;
_Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name);
DISPATCH_SAME_OPARG(); DISPATCH_SAME_OPARG();
} }
STAT_INC(LOAD_GLOBAL, deferred); STAT_INC(LOAD_GLOBAL, deferred);
DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */ #endif /* ENABLE_SPECIALIZATION */
} }
@ -1711,7 +1712,7 @@ dummy_func(
inst(INSTRUMENTED_LOAD_SUPER_ATTR, (unused/1, unused, unused, unused -- unused, unused if (oparg & 1))) { inst(INSTRUMENTED_LOAD_SUPER_ATTR, (unused/1, unused, unused, unused -- unused, unused if (oparg & 1))) {
// cancel out the decrement that will happen in LOAD_SUPER_ATTR; we // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we
// don't want to specialize instrumented instructions // don't want to specialize instrumented instructions
INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter);
GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); GO_TO_INSTRUCTION(LOAD_SUPER_ATTR);
} }
@ -1723,13 +1724,13 @@ dummy_func(
specializing op(_SPECIALIZE_LOAD_SUPER_ATTR, (counter/1, global_super, class, unused -- global_super, class, unused)) { specializing op(_SPECIALIZE_LOAD_SUPER_ATTR, (counter/1, global_super, class, unused -- global_super, class, unused)) {
#if ENABLE_SPECIALIZATION #if ENABLE_SPECIALIZATION
int load_method = oparg & 1; int load_method = oparg & 1;
if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr; next_instr = this_instr;
_Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method); _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method);
DISPATCH_SAME_OPARG(); DISPATCH_SAME_OPARG();
} }
STAT_INC(LOAD_SUPER_ATTR, deferred); STAT_INC(LOAD_SUPER_ATTR, deferred);
DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */ #endif /* ENABLE_SPECIALIZATION */
} }
@ -1836,14 +1837,14 @@ dummy_func(
specializing op(_SPECIALIZE_LOAD_ATTR, (counter/1, owner -- owner)) { specializing op(_SPECIALIZE_LOAD_ATTR, (counter/1, owner -- owner)) {
#if ENABLE_SPECIALIZATION #if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
next_instr = this_instr; next_instr = this_instr;
_Py_Specialize_LoadAttr(owner, next_instr, name); _Py_Specialize_LoadAttr(owner, next_instr, name);
DISPATCH_SAME_OPARG(); DISPATCH_SAME_OPARG();
} }
STAT_INC(LOAD_ATTR, deferred); STAT_INC(LOAD_ATTR, deferred);
DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */ #endif /* ENABLE_SPECIALIZATION */
} }
@ -2157,13 +2158,13 @@ dummy_func(
specializing op(_SPECIALIZE_COMPARE_OP, (counter/1, left, right -- left, right)) { specializing op(_SPECIALIZE_COMPARE_OP, (counter/1, left, right -- left, right)) {
#if ENABLE_SPECIALIZATION #if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr; next_instr = this_instr;
_Py_Specialize_CompareOp(left, right, next_instr, oparg); _Py_Specialize_CompareOp(left, right, next_instr, oparg);
DISPATCH_SAME_OPARG(); DISPATCH_SAME_OPARG();
} }
STAT_INC(COMPARE_OP, deferred); STAT_INC(COMPARE_OP, deferred);
DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */ #endif /* ENABLE_SPECIALIZATION */
} }
@ -2254,13 +2255,13 @@ dummy_func(
specializing op(_SPECIALIZE_CONTAINS_OP, (counter/1, left, right -- left, right)) { specializing op(_SPECIALIZE_CONTAINS_OP, (counter/1, left, right -- left, right)) {
#if ENABLE_SPECIALIZATION #if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr; next_instr = this_instr;
_Py_Specialize_ContainsOp(right, next_instr); _Py_Specialize_ContainsOp(right, next_instr);
DISPATCH_SAME_OPARG(); DISPATCH_SAME_OPARG();
} }
STAT_INC(CONTAINS_OP, deferred); STAT_INC(CONTAINS_OP, deferred);
DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */ #endif /* ENABLE_SPECIALIZATION */
} }
@ -2340,16 +2341,8 @@ dummy_func(
assert(oparg <= INSTR_OFFSET()); assert(oparg <= INSTR_OFFSET());
JUMPBY(-oparg); JUMPBY(-oparg);
#if ENABLE_SPECIALIZATION #if ENABLE_SPECIALIZATION
uint16_t counter = this_instr[1].cache; _Py_BackoffCounter counter = this_instr[1].counter;
this_instr[1].cache = counter + (1 << OPTIMIZER_BITS_IN_COUNTER); if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD) {
/* We are using unsigned values, but we really want signed values, so
* do the 2s complement adjustment manually */
uint32_t offset_counter = counter ^ (1 << 15);
uint32_t threshold = tstate->interp->optimizer_backedge_threshold;
assert((threshold & OPTIMIZER_BITS_MASK) == 0);
// Use '>=' not '>' so that the optimizer/backoff bits do not effect the result.
// Double-check that the opcode isn't instrumented or something:
if (offset_counter >= threshold && this_instr->op.code == JUMP_BACKWARD) {
_Py_CODEUNIT *start = this_instr; _Py_CODEUNIT *start = this_instr;
/* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */ /* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */
while (oparg > 255) { while (oparg > 255) {
@ -2365,16 +2358,11 @@ dummy_func(
GOTO_TIER_TWO(executor); GOTO_TIER_TWO(executor);
} }
else { else {
int backoff = this_instr[1].cache & OPTIMIZER_BITS_MASK; this_instr[1].counter = restart_backoff_counter(counter);
backoff++;
if (backoff < MIN_TIER2_BACKOFF) {
backoff = MIN_TIER2_BACKOFF;
} }
else if (backoff > MAX_TIER2_BACKOFF) {
backoff = MAX_TIER2_BACKOFF;
}
this_instr[1].cache = ((UINT16_MAX << OPTIMIZER_BITS_IN_COUNTER) << backoff) | backoff;
} }
else {
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
} }
#endif /* ENABLE_SPECIALIZATION */ #endif /* ENABLE_SPECIALIZATION */
} }
@ -2535,13 +2523,13 @@ dummy_func(
specializing op(_SPECIALIZE_FOR_ITER, (counter/1, iter -- iter)) { specializing op(_SPECIALIZE_FOR_ITER, (counter/1, iter -- iter)) {
#if ENABLE_SPECIALIZATION #if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr; next_instr = this_instr;
_Py_Specialize_ForIter(iter, next_instr, oparg); _Py_Specialize_ForIter(iter, next_instr, oparg);
DISPATCH_SAME_OPARG(); DISPATCH_SAME_OPARG();
} }
STAT_INC(FOR_ITER, deferred); STAT_INC(FOR_ITER, deferred);
DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */ #endif /* ENABLE_SPECIALIZATION */
} }
@ -3001,7 +2989,7 @@ dummy_func(
tstate, PY_MONITORING_EVENT_CALL, tstate, PY_MONITORING_EVENT_CALL,
frame, this_instr, function, arg); frame, this_instr, function, arg);
ERROR_IF(err, error); ERROR_IF(err, error);
INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter);
GO_TO_INSTRUCTION(CALL); GO_TO_INSTRUCTION(CALL);
} }
@ -3030,13 +3018,13 @@ dummy_func(
specializing op(_SPECIALIZE_CALL, (counter/1, callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { specializing op(_SPECIALIZE_CALL, (counter/1, callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) {
#if ENABLE_SPECIALIZATION #if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr; next_instr = this_instr;
_Py_Specialize_Call(callable, next_instr, oparg + (self_or_null != NULL)); _Py_Specialize_Call(callable, next_instr, oparg + (self_or_null != NULL));
DISPATCH_SAME_OPARG(); DISPATCH_SAME_OPARG();
} }
STAT_INC(CALL, deferred); STAT_INC(CALL, deferred);
DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */ #endif /* ENABLE_SPECIALIZATION */
} }
@ -3933,13 +3921,13 @@ dummy_func(
specializing op(_SPECIALIZE_BINARY_OP, (counter/1, lhs, rhs -- lhs, rhs)) { specializing op(_SPECIALIZE_BINARY_OP, (counter/1, lhs, rhs -- lhs, rhs)) {
#if ENABLE_SPECIALIZATION #if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr; next_instr = this_instr;
_Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY);
DISPATCH_SAME_OPARG(); DISPATCH_SAME_OPARG();
} }
STAT_INC(BINARY_OP, deferred); STAT_INC(BINARY_OP, deferred);
DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */ #endif /* ENABLE_SPECIALIZATION */
assert(NB_ADD <= oparg); assert(NB_ADD <= oparg);
assert(oparg <= NB_INPLACE_XOR); assert(oparg <= NB_INPLACE_XOR);
@ -3965,7 +3953,7 @@ dummy_func(
ERROR_IF(next_opcode < 0, error); ERROR_IF(next_opcode < 0, error);
next_instr = this_instr; next_instr = this_instr;
if (_PyOpcode_Caches[next_opcode]) { if (_PyOpcode_Caches[next_opcode]) {
INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); PAUSE_ADAPTIVE_COUNTER(next_instr[1].counter);
} }
assert(next_opcode > 0 && next_opcode < 256); assert(next_opcode > 0 && next_opcode < 256);
opcode = next_opcode; opcode = next_opcode;
@ -4157,21 +4145,22 @@ dummy_func(
tier2 op(_COLD_EXIT, (--)) { tier2 op(_COLD_EXIT, (--)) {
_PyExecutorObject *previous = (_PyExecutorObject *)tstate->previous_executor; _PyExecutorObject *previous = (_PyExecutorObject *)tstate->previous_executor;
_PyExitData *exit = &previous->exits[oparg]; _PyExitData *exit = &previous->exits[oparg];
exit->temperature++;
PyCodeObject *code = _PyFrame_GetCode(frame); PyCodeObject *code = _PyFrame_GetCode(frame);
_Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target; _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target;
if (exit->temperature < (int32_t)tstate->interp->optimizer_side_threshold) { _Py_BackoffCounter temperature = exit->temperature;
if (!backoff_counter_triggers(temperature)) {
exit->temperature = advance_backoff_counter(temperature);
GOTO_TIER_ONE(target); GOTO_TIER_ONE(target);
} }
_PyExecutorObject *executor; _PyExecutorObject *executor;
if (target->op.code == ENTER_EXECUTOR) { if (target->op.code == ENTER_EXECUTOR) {
executor = code->co_executors->executors[target->op.arg]; executor = code->co_executors->executors[target->op.arg];
Py_INCREF(executor); Py_INCREF(executor);
} else { }
else {
int optimized = _PyOptimizer_Optimize(frame, target, stack_pointer, &executor); int optimized = _PyOptimizer_Optimize(frame, target, stack_pointer, &executor);
if (optimized <= 0) { if (optimized <= 0) {
int32_t new_temp = -1 * tstate->interp->optimizer_side_threshold; exit->temperature = restart_backoff_counter(temperature);
exit->temperature = (new_temp < INT16_MIN) ? INT16_MIN : new_temp;
if (optimized < 0) { if (optimized < 0) {
Py_DECREF(previous); Py_DECREF(previous);
tstate->previous_executor = Py_None; tstate->previous_executor = Py_None;

5
Python/ceval.c
View File

@ -4,6 +4,7 @@
#include "Python.h" #include "Python.h"
#include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_abstract.h" // _PyIndex_Check()
#include "pycore_backoff.h"
#include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_call.h" // _PyObject_CallNoArgs()
#include "pycore_cell.h" // PyCell_GetRef() #include "pycore_cell.h" // PyCell_GetRef()
#include "pycore_ceval.h" #include "pycore_ceval.h"
@ -822,7 +823,7 @@ resume_frame:
_PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1);
/* Prevent the underlying instruction from specializing /* Prevent the underlying instruction from specializing
* and overwriting the instrumentation. */ * and overwriting the instrumentation. */
INCREMENT_ADAPTIVE_COUNTER(cache->counter); PAUSE_ADAPTIVE_COUNTER(cache->counter);
} }
opcode = original_opcode; opcode = original_opcode;
DISPATCH_GOTO(); DISPATCH_GOTO();
@ -1099,7 +1100,7 @@ exit_to_trace:
printf("SIDE EXIT: [UOp "); printf("SIDE EXIT: [UOp ");
_PyUOpPrint(&next_uop[-1]); _PyUOpPrint(&next_uop[-1]);
printf(", exit %u, temp %d, target %d -> %s]\n", printf(", exit %u, temp %d, target %d -> %s]\n",
exit_index, exit->temperature, exit->target, exit_index, exit->temperature.as_counter, exit->target,
_PyOpcode_OpName[_PyCode_CODE(_PyFrame_GetCode(frame))[exit->target].op.code]); _PyOpcode_OpName[_PyCode_CODE(_PyFrame_GetCode(frame))[exit->target].op.code]);
} }
#endif #endif

23
Python/ceval_macros.h
View File

@ -262,7 +262,7 @@ GETITEM(PyObject *v, Py_ssize_t i) {
STAT_INC(opcode, miss); \ STAT_INC(opcode, miss); \
STAT_INC((INSTNAME), miss); \ STAT_INC((INSTNAME), miss); \
/* The counter is always the first cache entry: */ \ /* The counter is always the first cache entry: */ \
if (ADAPTIVE_COUNTER_IS_ZERO(next_instr->cache)) { \ if (ADAPTIVE_COUNTER_TRIGGERS(next_instr->cache)) { \
STAT_INC((INSTNAME), deopt); \ STAT_INC((INSTNAME), deopt); \
} \ } \
} while (0) } while (0)
@ -290,29 +290,28 @@ GETITEM(PyObject *v, Py_ssize_t i) {
dtrace_function_entry(frame); \ dtrace_function_entry(frame); \
} }
#define ADAPTIVE_COUNTER_IS_ZERO(COUNTER) \ /* This takes a uint16_t instead of a _Py_BackoffCounter,
(((COUNTER) >> ADAPTIVE_BACKOFF_BITS) == 0) * because it is used directly on the cache entry in generated code,
* which is always an integral type. */
#define ADAPTIVE_COUNTER_IS_MAX(COUNTER) \ #define ADAPTIVE_COUNTER_TRIGGERS(COUNTER) \
(((COUNTER) >> ADAPTIVE_BACKOFF_BITS) == ((1 << MAX_BACKOFF_VALUE) - 1)) backoff_counter_triggers(forge_backoff_counter((COUNTER)))
#ifdef Py_GIL_DISABLED #ifdef Py_GIL_DISABLED
#define DECREMENT_ADAPTIVE_COUNTER(COUNTER) \ #define ADVANCE_ADAPTIVE_COUNTER(COUNTER) \
do { \ do { \
/* gh-115999 tracks progress on addressing this. */ \ /* gh-115999 tracks progress on addressing this. */ \
static_assert(0, "The specializing interpreter is not yet thread-safe"); \ static_assert(0, "The specializing interpreter is not yet thread-safe"); \
} while (0); } while (0);
#else #else
#define DECREMENT_ADAPTIVE_COUNTER(COUNTER) \ #define ADVANCE_ADAPTIVE_COUNTER(COUNTER) \
do { \ do { \
assert(!ADAPTIVE_COUNTER_IS_ZERO((COUNTER))); \ (COUNTER) = advance_backoff_counter((COUNTER)); \
(COUNTER) -= (1 << ADAPTIVE_BACKOFF_BITS); \
} while (0); } while (0);
#endif #endif
#define INCREMENT_ADAPTIVE_COUNTER(COUNTER) \ #define PAUSE_ADAPTIVE_COUNTER(COUNTER) \
do { \ do { \
(COUNTER) += (1 << ADAPTIVE_BACKOFF_BITS); \ (COUNTER) = pause_backoff_counter((COUNTER)); \
} while (0); } while (0);
#define UNBOUNDLOCAL_ERROR_MSG \ #define UNBOUNDLOCAL_ERROR_MSG \

11
Python/executor_cases.c.h generated
View File

@ -3694,21 +3694,22 @@
oparg = CURRENT_OPARG(); oparg = CURRENT_OPARG();
_PyExecutorObject *previous = (_PyExecutorObject *)tstate->previous_executor; _PyExecutorObject *previous = (_PyExecutorObject *)tstate->previous_executor;
_PyExitData *exit = &previous->exits[oparg]; _PyExitData *exit = &previous->exits[oparg];
exit->temperature++;
PyCodeObject *code = _PyFrame_GetCode(frame); PyCodeObject *code = _PyFrame_GetCode(frame);
_Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target; _Py_CODEUNIT *target = _PyCode_CODE(code) + exit->target;
if (exit->temperature < (int32_t)tstate->interp->optimizer_side_threshold) { _Py_BackoffCounter temperature = exit->temperature;
if (!backoff_counter_triggers(temperature)) {
exit->temperature = advance_backoff_counter(temperature);
GOTO_TIER_ONE(target); GOTO_TIER_ONE(target);
} }
_PyExecutorObject *executor; _PyExecutorObject *executor;
if (target->op.code == ENTER_EXECUTOR) { if (target->op.code == ENTER_EXECUTOR) {
executor = code->co_executors->executors[target->op.arg]; executor = code->co_executors->executors[target->op.arg];
Py_INCREF(executor); Py_INCREF(executor);
} else { }
else {
int optimized = _PyOptimizer_Optimize(frame, target, stack_pointer, &executor); int optimized = _PyOptimizer_Optimize(frame, target, stack_pointer, &executor);
if (optimized <= 0) { if (optimized <= 0) {
int32_t new_temp = -1 * tstate->interp->optimizer_side_threshold; exit->temperature = restart_backoff_counter(temperature);
exit->temperature = (new_temp < INT16_MIN) ? INT16_MIN : new_temp;
if (optimized < 0) { if (optimized < 0) {
Py_DECREF(previous); Py_DECREF(previous);
tstate->previous_executor = Py_None; tstate->previous_executor = Py_None;

85
Python/generated_cases.c.h generated
View File

@ -115,13 +115,13 @@
uint16_t counter = read_u16(&this_instr[1].cache); uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter; (void)counter;
#if ENABLE_SPECIALIZATION #if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr; next_instr = this_instr;
_Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY);
DISPATCH_SAME_OPARG(); DISPATCH_SAME_OPARG();
} }
STAT_INC(BINARY_OP, deferred); STAT_INC(BINARY_OP, deferred);
DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */ #endif /* ENABLE_SPECIALIZATION */
assert(NB_ADD <= oparg); assert(NB_ADD <= oparg);
assert(oparg <= NB_INPLACE_XOR); assert(oparg <= NB_INPLACE_XOR);
@ -432,13 +432,13 @@
uint16_t counter = read_u16(&this_instr[1].cache); uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter; (void)counter;
#if ENABLE_SPECIALIZATION #if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr; next_instr = this_instr;
_Py_Specialize_BinarySubscr(container, sub, next_instr); _Py_Specialize_BinarySubscr(container, sub, next_instr);
DISPATCH_SAME_OPARG(); DISPATCH_SAME_OPARG();
} }
STAT_INC(BINARY_SUBSCR, deferred); STAT_INC(BINARY_SUBSCR, deferred);
DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */ #endif /* ENABLE_SPECIALIZATION */
} }
// _BINARY_SUBSCR // _BINARY_SUBSCR
@ -760,13 +760,13 @@
uint16_t counter = read_u16(&this_instr[1].cache); uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter; (void)counter;
#if ENABLE_SPECIALIZATION #if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr; next_instr = this_instr;
_Py_Specialize_Call(callable, next_instr, oparg + (self_or_null != NULL)); _Py_Specialize_Call(callable, next_instr, oparg + (self_or_null != NULL));
DISPATCH_SAME_OPARG(); DISPATCH_SAME_OPARG();
} }
STAT_INC(CALL, deferred); STAT_INC(CALL, deferred);
DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */ #endif /* ENABLE_SPECIALIZATION */
} }
/* Skip 2 cache entries */ /* Skip 2 cache entries */
@ -2036,13 +2036,13 @@
uint16_t counter = read_u16(&this_instr[1].cache); uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter; (void)counter;
#if ENABLE_SPECIALIZATION #if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr; next_instr = this_instr;
_Py_Specialize_CompareOp(left, right, next_instr, oparg); _Py_Specialize_CompareOp(left, right, next_instr, oparg);
DISPATCH_SAME_OPARG(); DISPATCH_SAME_OPARG();
} }
STAT_INC(COMPARE_OP, deferred); STAT_INC(COMPARE_OP, deferred);
DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */ #endif /* ENABLE_SPECIALIZATION */
} }
// _COMPARE_OP // _COMPARE_OP
@ -2185,13 +2185,13 @@
uint16_t counter = read_u16(&this_instr[1].cache); uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter; (void)counter;
#if ENABLE_SPECIALIZATION #if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr; next_instr = this_instr;
_Py_Specialize_ContainsOp(right, next_instr); _Py_Specialize_ContainsOp(right, next_instr);
DISPATCH_SAME_OPARG(); DISPATCH_SAME_OPARG();
} }
STAT_INC(CONTAINS_OP, deferred); STAT_INC(CONTAINS_OP, deferred);
DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */ #endif /* ENABLE_SPECIALIZATION */
} }
// _CONTAINS_OP // _CONTAINS_OP
@ -2596,13 +2596,13 @@
uint16_t counter = read_u16(&this_instr[1].cache); uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter; (void)counter;
#if ENABLE_SPECIALIZATION #if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr; next_instr = this_instr;
_Py_Specialize_ForIter(iter, next_instr, oparg); _Py_Specialize_ForIter(iter, next_instr, oparg);
DISPATCH_SAME_OPARG(); DISPATCH_SAME_OPARG();
} }
STAT_INC(FOR_ITER, deferred); STAT_INC(FOR_ITER, deferred);
DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */ #endif /* ENABLE_SPECIALIZATION */
} }
// _FOR_ITER // _FOR_ITER
@ -3026,7 +3026,7 @@
tstate, PY_MONITORING_EVENT_CALL, tstate, PY_MONITORING_EVENT_CALL,
frame, this_instr, function, arg); frame, this_instr, function, arg);
if (err) goto error; if (err) goto error;
INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter);
GO_TO_INSTRUCTION(CALL); GO_TO_INSTRUCTION(CALL);
} }
@ -3142,7 +3142,7 @@
if (next_opcode < 0) goto error; if (next_opcode < 0) goto error;
next_instr = this_instr; next_instr = this_instr;
if (_PyOpcode_Caches[next_opcode]) { if (_PyOpcode_Caches[next_opcode]) {
INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); PAUSE_ADAPTIVE_COUNTER(next_instr[1].counter);
} }
assert(next_opcode > 0 && next_opcode < 256); assert(next_opcode > 0 && next_opcode < 256);
opcode = next_opcode; opcode = next_opcode;
@ -3177,7 +3177,7 @@
/* Skip 1 cache entry */ /* Skip 1 cache entry */
// cancel out the decrement that will happen in LOAD_SUPER_ATTR; we // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we
// don't want to specialize instrumented instructions // don't want to specialize instrumented instructions
INCREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter);
GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); GO_TO_INSTRUCTION(LOAD_SUPER_ATTR);
} }
@ -3415,16 +3415,8 @@
assert(oparg <= INSTR_OFFSET()); assert(oparg <= INSTR_OFFSET());
JUMPBY(-oparg); JUMPBY(-oparg);
#if ENABLE_SPECIALIZATION #if ENABLE_SPECIALIZATION
uint16_t counter = this_instr[1].cache; _Py_BackoffCounter counter = this_instr[1].counter;
this_instr[1].cache = counter + (1 << OPTIMIZER_BITS_IN_COUNTER); if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD) {
/* We are using unsigned values, but we really want signed values, so
* do the 2s complement adjustment manually */
uint32_t offset_counter = counter ^ (1 << 15);
uint32_t threshold = tstate->interp->optimizer_backedge_threshold;
assert((threshold & OPTIMIZER_BITS_MASK) == 0);
// Use '>=' not '>' so that the optimizer/backoff bits do not effect the result.
// Double-check that the opcode isn't instrumented or something:
if (offset_counter >= threshold && this_instr->op.code == JUMP_BACKWARD) {
_Py_CODEUNIT *start = this_instr; _Py_CODEUNIT *start = this_instr;
/* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */ /* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */
while (oparg > 255) { while (oparg > 255) {
@ -3440,16 +3432,11 @@
GOTO_TIER_TWO(executor); GOTO_TIER_TWO(executor);
} }
else { else {
int backoff = this_instr[1].cache & OPTIMIZER_BITS_MASK; this_instr[1].counter = restart_backoff_counter(counter);
backoff++;
if (backoff < MIN_TIER2_BACKOFF) {
backoff = MIN_TIER2_BACKOFF;
} }
else if (backoff > MAX_TIER2_BACKOFF) {
backoff = MAX_TIER2_BACKOFF;
}
this_instr[1].cache = ((UINT16_MAX << OPTIMIZER_BITS_IN_COUNTER) << backoff) | backoff;
} }
else {
ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
} }
#endif /* ENABLE_SPECIALIZATION */ #endif /* ENABLE_SPECIALIZATION */
DISPATCH(); DISPATCH();
@ -3543,14 +3530,14 @@
uint16_t counter = read_u16(&this_instr[1].cache); uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter; (void)counter;
#if ENABLE_SPECIALIZATION #if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
next_instr = this_instr; next_instr = this_instr;
_Py_Specialize_LoadAttr(owner, next_instr, name); _Py_Specialize_LoadAttr(owner, next_instr, name);
DISPATCH_SAME_OPARG(); DISPATCH_SAME_OPARG();
} }
STAT_INC(LOAD_ATTR, deferred); STAT_INC(LOAD_ATTR, deferred);
DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */ #endif /* ENABLE_SPECIALIZATION */
} }
/* Skip 8 cache entries */ /* Skip 8 cache entries */
@ -4238,14 +4225,14 @@
uint16_t counter = read_u16(&this_instr[1].cache); uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter; (void)counter;
#if ENABLE_SPECIALIZATION #if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
next_instr = this_instr; next_instr = this_instr;
_Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name);
DISPATCH_SAME_OPARG(); DISPATCH_SAME_OPARG();
} }
STAT_INC(LOAD_GLOBAL, deferred); STAT_INC(LOAD_GLOBAL, deferred);
DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */ #endif /* ENABLE_SPECIALIZATION */
} }
/* Skip 1 cache entry */ /* Skip 1 cache entry */
@ -4442,13 +4429,13 @@
(void)counter; (void)counter;
#if ENABLE_SPECIALIZATION #if ENABLE_SPECIALIZATION
int load_method = oparg & 1; int load_method = oparg & 1;
if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr; next_instr = this_instr;
_Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method); _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method);
DISPATCH_SAME_OPARG(); DISPATCH_SAME_OPARG();
} }
STAT_INC(LOAD_SUPER_ATTR, deferred); STAT_INC(LOAD_SUPER_ATTR, deferred);
DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */ #endif /* ENABLE_SPECIALIZATION */
} }
// _LOAD_SUPER_ATTR // _LOAD_SUPER_ATTR
@ -5083,13 +5070,13 @@
uint16_t counter = read_u16(&this_instr[1].cache); uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter; (void)counter;
#if ENABLE_SPECIALIZATION #if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr; next_instr = this_instr;
_Py_Specialize_Send(receiver, next_instr); _Py_Specialize_Send(receiver, next_instr);
DISPATCH_SAME_OPARG(); DISPATCH_SAME_OPARG();
} }
STAT_INC(SEND, deferred); STAT_INC(SEND, deferred);
DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */ #endif /* ENABLE_SPECIALIZATION */
} }
// _SEND // _SEND
@ -5271,14 +5258,14 @@
uint16_t counter = read_u16(&this_instr[1].cache); uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter; (void)counter;
#if ENABLE_SPECIALIZATION #if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
next_instr = this_instr; next_instr = this_instr;
_Py_Specialize_StoreAttr(owner, next_instr, name); _Py_Specialize_StoreAttr(owner, next_instr, name);
DISPATCH_SAME_OPARG(); DISPATCH_SAME_OPARG();
} }
STAT_INC(STORE_ATTR, deferred); STAT_INC(STORE_ATTR, deferred);
DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */ #endif /* ENABLE_SPECIALIZATION */
} }
/* Skip 3 cache entries */ /* Skip 3 cache entries */
@ -5562,13 +5549,13 @@
uint16_t counter = read_u16(&this_instr[1].cache); uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter; (void)counter;
#if ENABLE_SPECIALIZATION #if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr; next_instr = this_instr;
_Py_Specialize_StoreSubscr(container, sub, next_instr); _Py_Specialize_StoreSubscr(container, sub, next_instr);
DISPATCH_SAME_OPARG(); DISPATCH_SAME_OPARG();
} }
STAT_INC(STORE_SUBSCR, deferred); STAT_INC(STORE_SUBSCR, deferred);
DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */ #endif /* ENABLE_SPECIALIZATION */
} }
// _STORE_SUBSCR // _STORE_SUBSCR
@ -5665,13 +5652,13 @@
uint16_t counter = read_u16(&this_instr[1].cache); uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter; (void)counter;
#if ENABLE_SPECIALIZATION #if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr; next_instr = this_instr;
_Py_Specialize_ToBool(value, next_instr); _Py_Specialize_ToBool(value, next_instr);
DISPATCH_SAME_OPARG(); DISPATCH_SAME_OPARG();
} }
STAT_INC(TO_BOOL, deferred); STAT_INC(TO_BOOL, deferred);
DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */ #endif /* ENABLE_SPECIALIZATION */
} }
/* Skip 2 cache entries */ /* Skip 2 cache entries */
@ -5882,13 +5869,13 @@
uint16_t counter = read_u16(&this_instr[1].cache); uint16_t counter = read_u16(&this_instr[1].cache);
(void)counter; (void)counter;
#if ENABLE_SPECIALIZATION #if ENABLE_SPECIALIZATION
if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
next_instr = this_instr; next_instr = this_instr;
_Py_Specialize_UnpackSequence(seq, next_instr, oparg); _Py_Specialize_UnpackSequence(seq, next_instr, oparg);
DISPATCH_SAME_OPARG(); DISPATCH_SAME_OPARG();
} }
STAT_INC(UNPACK_SEQUENCE, deferred); STAT_INC(UNPACK_SEQUENCE, deferred);
DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
#endif /* ENABLE_SPECIALIZATION */ #endif /* ENABLE_SPECIALIZATION */
(void)seq; (void)seq;
(void)counter; (void)counter;

8
Python/instrumentation.c
View File

@ -590,7 +590,7 @@ de_instrument(PyCodeObject *code, int i, int event)
CHECK(_PyOpcode_Deopt[deinstrumented] == deinstrumented); CHECK(_PyOpcode_Deopt[deinstrumented] == deinstrumented);
*opcode_ptr = deinstrumented; *opcode_ptr = deinstrumented;
if (_PyOpcode_Caches[deinstrumented]) { if (_PyOpcode_Caches[deinstrumented]) {
instr[1].cache = adaptive_counter_warmup(); instr[1].counter = adaptive_counter_warmup();
} }
} }
@ -611,7 +611,7 @@ de_instrument_line(PyCodeObject *code, int i)
CHECK(original_opcode == _PyOpcode_Deopt[original_opcode]); CHECK(original_opcode == _PyOpcode_Deopt[original_opcode]);
instr->op.code = original_opcode; instr->op.code = original_opcode;
if (_PyOpcode_Caches[original_opcode]) { if (_PyOpcode_Caches[original_opcode]) {
instr[1].cache = adaptive_counter_warmup(); instr[1].counter = adaptive_counter_warmup();
} }
assert(instr->op.code != INSTRUMENTED_LINE); assert(instr->op.code != INSTRUMENTED_LINE);
} }
@ -634,7 +634,7 @@ de_instrument_per_instruction(PyCodeObject *code, int i)
CHECK(original_opcode == _PyOpcode_Deopt[original_opcode]); CHECK(original_opcode == _PyOpcode_Deopt[original_opcode]);
*opcode_ptr = original_opcode; *opcode_ptr = original_opcode;
if (_PyOpcode_Caches[original_opcode]) { if (_PyOpcode_Caches[original_opcode]) {
instr[1].cache = adaptive_counter_warmup(); instr[1].counter = adaptive_counter_warmup();
} }
assert(*opcode_ptr != INSTRUMENTED_INSTRUCTION); assert(*opcode_ptr != INSTRUMENTED_INSTRUCTION);
assert(instr->op.code != INSTRUMENTED_INSTRUCTION); assert(instr->op.code != INSTRUMENTED_INSTRUCTION);
@ -667,7 +667,7 @@ instrument(PyCodeObject *code, int i)
assert(instrumented); assert(instrumented);
*opcode_ptr = instrumented; *opcode_ptr = instrumented;
if (_PyOpcode_Caches[deopt]) { if (_PyOpcode_Caches[deopt]) {
instr[1].cache = adaptive_counter_warmup(); instr[1].counter = adaptive_counter_warmup();
} }
} }
} }

37
Python/optimizer.c
View File

@ -1,6 +1,7 @@
#include "Python.h" #include "Python.h"
#include "opcode.h" #include "opcode.h"
#include "pycore_interp.h" #include "pycore_interp.h"
#include "pycore_backoff.h"
#include "pycore_bitutils.h" // _Py_popcount32() #include "pycore_bitutils.h" // _Py_popcount32()
#include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_object.h" // _PyObject_GC_UNTRACK()
#include "pycore_opcode_metadata.h" // _PyOpcode_OpName[] #include "pycore_opcode_metadata.h" // _PyOpcode_OpName[]
@ -110,9 +111,7 @@ never_optimize(
_PyExecutorObject **exec, _PyExecutorObject **exec,
int Py_UNUSED(stack_entries)) int Py_UNUSED(stack_entries))
{ {
/* Although it should be benign for this to be called, // This may be called if the optimizer is reset
* it shouldn't happen, so fail in debug builds. */
assert(0 && "never optimize should never be called");
return 0; return 0;
} }
@ -127,25 +126,12 @@ PyTypeObject _PyDefaultOptimizer_Type = {
static _PyOptimizerObject _PyOptimizer_Default = { static _PyOptimizerObject _PyOptimizer_Default = {
PyObject_HEAD_INIT(&_PyDefaultOptimizer_Type) PyObject_HEAD_INIT(&_PyDefaultOptimizer_Type)
.optimize = never_optimize, .optimize = never_optimize,
.resume_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD,
.backedge_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD,
.side_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD,
}; };
static uint32_t
shift_and_offset_threshold(uint32_t threshold)
{
return (threshold << OPTIMIZER_BITS_IN_COUNTER) + (1 << 15);
}
_PyOptimizerObject * _PyOptimizerObject *
PyUnstable_GetOptimizer(void) PyUnstable_GetOptimizer(void)
{ {
PyInterpreterState *interp = _PyInterpreterState_GET(); PyInterpreterState *interp = _PyInterpreterState_GET();
assert(interp->optimizer_backedge_threshold ==
shift_and_offset_threshold(interp->optimizer->backedge_threshold));
assert(interp->optimizer_resume_threshold ==
shift_and_offset_threshold(interp->optimizer->resume_threshold));
if (interp->optimizer == &_PyOptimizer_Default) { if (interp->optimizer == &_PyOptimizer_Default) {
return NULL; return NULL;
} }
@ -190,13 +176,6 @@ _Py_SetOptimizer(PyInterpreterState *interp, _PyOptimizerObject *optimizer)
} }
Py_INCREF(optimizer); Py_INCREF(optimizer);
interp->optimizer = optimizer; interp->optimizer = optimizer;
interp->optimizer_backedge_threshold = shift_and_offset_threshold(optimizer->backedge_threshold);
interp->optimizer_resume_threshold = shift_and_offset_threshold(optimizer->resume_threshold);
interp->optimizer_side_threshold = optimizer->side_threshold;
if (optimizer == &_PyOptimizer_Default) {
assert(interp->optimizer_backedge_threshold > (1 << 16));
assert(interp->optimizer_resume_threshold > (1 << 16));
}
return old; return old;
} }
@ -1109,7 +1088,7 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil
assert(exit_count < COLD_EXIT_COUNT); assert(exit_count < COLD_EXIT_COUNT);
for (int i = 0; i < exit_count; i++) { for (int i = 0; i < exit_count; i++) {
executor->exits[i].executor = &COLD_EXITS[i]; executor->exits[i].executor = &COLD_EXITS[i];
executor->exits[i].temperature = 0; executor->exits[i].temperature = initial_temperature_backoff_counter();
} }
int next_exit = exit_count-1; int next_exit = exit_count-1;
_PyUOpInstruction *dest = (_PyUOpInstruction *)&executor->trace[length]; _PyUOpInstruction *dest = (_PyUOpInstruction *)&executor->trace[length];
@ -1291,11 +1270,6 @@ PyUnstable_Optimizer_NewUOpOptimizer(void)
return NULL; return NULL;
} }
opt->optimize = uop_optimize; opt->optimize = uop_optimize;
opt->resume_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD;
// Need a few iterations to settle specializations,
// and to ammortize the cost of optimization.
opt->side_threshold = 16;
opt->backedge_threshold = 16;
return (PyObject *)opt; return (PyObject *)opt;
} }
@ -1385,9 +1359,6 @@ PyUnstable_Optimizer_NewCounter(void)
return NULL; return NULL;
} }
opt->base.optimize = counter_optimize; opt->base.optimize = counter_optimize;
opt->base.resume_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD;
opt->base.side_threshold = OPTIMIZER_UNREACHABLE_THRESHOLD;
opt->base.backedge_threshold = 0;
opt->count = 0; opt->count = 0;
return (PyObject *)opt; return (PyObject *)opt;
} }
@ -1554,7 +1525,7 @@ _Py_ExecutorClear(_PyExecutorObject *executor)
for (uint32_t i = 0; i < executor->exit_count; i++) { for (uint32_t i = 0; i < executor->exit_count; i++) {
Py_DECREF(executor->exits[i].executor); Py_DECREF(executor->exits[i].executor);
executor->exits[i].executor = &COLD_EXITS[i]; executor->exits[i].executor = &COLD_EXITS[i];
executor->exits[i].temperature = INT16_MIN; executor->exits[i].temperature = initial_unreachable_backoff_counter();
} }
_Py_CODEUNIT *instruction = &_PyCode_CODE(code)[executor->vm_data.index]; _Py_CODEUNIT *instruction = &_PyCode_CODE(code)[executor->vm_data.index];
assert(instruction->op.code == ENTER_EXECUTOR); assert(instruction->op.code == ENTER_EXECUTOR);

8
Python/specialize.c
View File

@ -419,22 +419,20 @@ _PyCode_Quicken(PyCodeObject *code)
int caches = _PyOpcode_Caches[opcode]; int caches = _PyOpcode_Caches[opcode];
if (caches) { if (caches) {
// The initial value depends on the opcode // The initial value depends on the opcode
int initial_value;
switch (opcode) { switch (opcode) {
case JUMP_BACKWARD: case JUMP_BACKWARD:
initial_value = 0; instructions[i + 1].counter = initial_jump_backoff_counter();
break; break;
case POP_JUMP_IF_FALSE: case POP_JUMP_IF_FALSE:
case POP_JUMP_IF_TRUE: case POP_JUMP_IF_TRUE:
case POP_JUMP_IF_NONE: case POP_JUMP_IF_NONE:
case POP_JUMP_IF_NOT_NONE: case POP_JUMP_IF_NOT_NONE:
initial_value = 0x5555; // Alternating 0, 1 bits instructions[i + 1].cache = 0x5555; // Alternating 0, 1 bits
break; break;
default: default:
initial_value = adaptive_counter_warmup(); instructions[i + 1].counter = adaptive_counter_warmup();
break; break;
} }
instructions[i + 1].cache = initial_value;
i += caches; i += caches;
} }
} }

1
Tools/jit/template.c
View File

@ -1,5 +1,6 @@
#include "Python.h" #include "Python.h"
#include "pycore_backoff.h"
#include "pycore_call.h" #include "pycore_call.h"
#include "pycore_ceval.h" #include "pycore_ceval.h"
#include "pycore_cell.h" #include "pycore_cell.h"