From c179c0e6cbb4d1e981fffd43f207f5b1aa5388e5 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Wed, 17 Apr 2024 16:42:04 +0100 Subject: [PATCH] gh-117680: make _PyInstructionSequence a PyObject and use it in tests (#117629) --- .../pycore_global_objects_fini_generated.h | 4 + Include/internal/pycore_global_strings.h | 4 + .../internal/pycore_instruction_sequence.h | 16 +- .../internal/pycore_runtime_init_generated.h | 4 + .../internal/pycore_unicodeobject_generated.h | 12 + Lib/test/support/bytecode_helper.py | 85 +++-- Lib/test/test_compile.py | 46 +++ Lib/test/test_compiler_assemble.py | 20 +- Lib/test/test_peepholer.py | 48 +-- ...-04-09-16-07-00.gh-issue-117680.MRZ78K.rst | 1 + Modules/_testinternalcapi.c | 15 + Modules/clinic/_testinternalcapi.c.h | 20 +- Objects/object.c | 2 + Python/clinic/instruction_sequence.c.h | 304 ++++++++++++++++++ Python/compile.c | 198 ++---------- Python/instruction_sequence.c | 300 +++++++++++++++++ Tools/c-analyzer/cpython/globals-to-fix.tsv | 1 + 17 files changed, 838 insertions(+), 242 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-04-09-16-07-00.gh-issue-117680.MRZ78K.rst create mode 100644 Python/clinic/instruction_sequence.c.h diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 24f32fca2e5..90a338ade17 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -864,6 +864,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(co_stacksize)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(co_varnames)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(code)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(col_offset)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(command)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(comment_factory)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(compile_mode)); @@ -913,6 +914,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(encode)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(encoding)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(end)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(end_col_offset)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(end_lineno)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(end_offset)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(endpos)); @@ -1033,6 +1035,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(kw1)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(kw2)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(kwdefaults)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(label)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(lambda)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(last)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(last_exc)); @@ -1096,6 +1099,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(namespaces)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(narg)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ndigits)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(nested)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(new_file_name)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(new_limit)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(newline)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 024f817408c..0899e7ee776 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -353,6 +353,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(co_stacksize) STRUCT_FOR_ID(co_varnames) STRUCT_FOR_ID(code) + STRUCT_FOR_ID(col_offset) STRUCT_FOR_ID(command) STRUCT_FOR_ID(comment_factory) STRUCT_FOR_ID(compile_mode) @@ -402,6 +403,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(encode) STRUCT_FOR_ID(encoding) STRUCT_FOR_ID(end) + STRUCT_FOR_ID(end_col_offset) STRUCT_FOR_ID(end_lineno) STRUCT_FOR_ID(end_offset) STRUCT_FOR_ID(endpos) @@ -522,6 +524,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(kw1) STRUCT_FOR_ID(kw2) STRUCT_FOR_ID(kwdefaults) + STRUCT_FOR_ID(label) STRUCT_FOR_ID(lambda) STRUCT_FOR_ID(last) STRUCT_FOR_ID(last_exc) @@ -585,6 +588,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(namespaces) STRUCT_FOR_ID(narg) STRUCT_FOR_ID(ndigits) + STRUCT_FOR_ID(nested) STRUCT_FOR_ID(new_file_name) STRUCT_FOR_ID(new_limit) STRUCT_FOR_ID(newline) diff --git a/Include/internal/pycore_instruction_sequence.h b/Include/internal/pycore_instruction_sequence.h index b57484fa053..ecba0d9d8e9 100644 --- a/Include/internal/pycore_instruction_sequence.h +++ b/Include/internal/pycore_instruction_sequence.h @@ -5,10 +5,13 @@ # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_symtable.h" + #ifdef __cplusplus extern "C" { #endif + typedef struct { int h_label; int h_startdepth; @@ -26,23 +29,30 @@ typedef struct { int i_offset; } _PyInstruction; -typedef struct { +typedef struct instruction_sequence { + PyObject_HEAD _PyInstruction *s_instrs; int s_allocated; int s_used; int s_next_free_label; /* next free label id */ + /* Map of a label id to instruction offset (index into s_instrs). * If s_labelmap is NULL, then each label id is the offset itself. */ - int *s_labelmap; /* label id --> instr offset */ + int *s_labelmap; int s_labelmap_size; + + /* PyList of instruction sequences of nested functions */ + PyObject *s_nested; } _PyInstructionSequence; typedef struct { int id; } _PyJumpTargetLabel; +PyAPI_FUNC(PyObject*)_PyInstructionSequence_New(void); + int _PyInstructionSequence_UseLabel(_PyInstructionSequence *seq, int lbl); int _PyInstructionSequence_Addop(_PyInstructionSequence *seq, int opcode, int oparg, @@ -53,6 +63,8 @@ int _PyInstructionSequence_InsertInstruction(_PyInstructionSequence *seq, int po int opcode, int oparg, _Py_SourceLocation loc); void PyInstructionSequence_Fini(_PyInstructionSequence *seq); +extern PyTypeObject _PyInstructionSequence_Type; +#define _PyInstructionSequence_Check(v) Py_IS_TYPE((v), &_PyInstructionSequence_Type) #ifdef __cplusplus } diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 795f95cb3a7..d4323e5bd12 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -862,6 +862,7 @@ extern "C" { INIT_ID(co_stacksize), \ INIT_ID(co_varnames), \ INIT_ID(code), \ + INIT_ID(col_offset), \ INIT_ID(command), \ INIT_ID(comment_factory), \ INIT_ID(compile_mode), \ @@ -911,6 +912,7 @@ extern "C" { INIT_ID(encode), \ INIT_ID(encoding), \ INIT_ID(end), \ + INIT_ID(end_col_offset), \ INIT_ID(end_lineno), \ INIT_ID(end_offset), \ INIT_ID(endpos), \ @@ -1031,6 +1033,7 @@ extern "C" { INIT_ID(kw1), \ INIT_ID(kw2), \ INIT_ID(kwdefaults), \ + INIT_ID(label), \ INIT_ID(lambda), \ INIT_ID(last), \ INIT_ID(last_exc), \ @@ -1094,6 +1097,7 @@ extern "C" { INIT_ID(namespaces), \ INIT_ID(narg), \ INIT_ID(ndigits), \ + INIT_ID(nested), \ INIT_ID(new_file_name), \ INIT_ID(new_limit), \ INIT_ID(newline), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 272462e99b0..9daef267069 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -900,6 +900,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(code); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(col_offset); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(command); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1047,6 +1050,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(end); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(end_col_offset); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(end_lineno); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1407,6 +1413,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(kwdefaults); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(label); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(lambda); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1596,6 +1605,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(ndigits); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(nested); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(new_file_name); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Lib/test/support/bytecode_helper.py b/Lib/test/support/bytecode_helper.py index 7a0e884ccc1..85bcd1f0f1c 100644 --- a/Lib/test/support/bytecode_helper.py +++ b/Lib/test/support/bytecode_helper.py @@ -3,6 +3,7 @@ import unittest import dis import io +import opcode try: import _testinternalcapi except ImportError: @@ -68,16 +69,14 @@ class CompilationStepTestCase(unittest.TestCase): class Label: pass - def assertInstructionsMatch(self, actual_, expected_): - # get two lists where each entry is a label or - # an instruction tuple. Normalize the labels to the - # instruction count of the target, and compare the lists. + def assertInstructionsMatch(self, actual_seq, expected): + # get an InstructionSequence and an expected list, where each + # entry is a label or an instruction tuple. Construct an expcted + # instruction sequence and compare with the one given. - self.assertIsInstance(actual_, list) - self.assertIsInstance(expected_, list) - - actual = self.normalize_insts(actual_) - expected = self.normalize_insts(expected_) + self.assertIsInstance(expected, list) + actual = actual_seq.get_instructions() + expected = self.seq_from_insts(expected).get_instructions() self.assertEqual(len(actual), len(expected)) # compare instructions @@ -87,10 +86,8 @@ class CompilationStepTestCase(unittest.TestCase): continue self.assertIsInstance(exp, tuple) self.assertIsInstance(act, tuple) - # crop comparison to the provided expected values - if len(act) > len(exp): - act = act[:len(exp)] - self.assertEqual(exp, act) + idx = max([p[0] for p in enumerate(exp) if p[1] != -1]) + self.assertEqual(exp[:idx], act[:idx]) def resolveAndRemoveLabels(self, insts): idx = 0 @@ -105,35 +102,37 @@ class CompilationStepTestCase(unittest.TestCase): return res - def normalize_insts(self, insts): - """ Map labels to instruction index. - Map opcodes to opnames. - """ - insts = self.resolveAndRemoveLabels(insts) - res = [] - for item in insts: - assert isinstance(item, tuple) - opcode, oparg, *loc = item - opcode = dis.opmap.get(opcode, opcode) - if isinstance(oparg, self.Label): - arg = oparg.value - else: - arg = oparg if opcode in self.HAS_ARG else None - opcode = dis.opname[opcode] - res.append((opcode, arg, *loc)) - return res + def seq_from_insts(self, insts): + labels = {item for item in insts if isinstance(item, self.Label)} + for i, lbl in enumerate(labels): + lbl.value = i - def complete_insts_info(self, insts): - # fill in omitted fields in location, and oparg 0 for ops with no arg. - res = [] + seq = _testinternalcapi.new_instruction_sequence() for item in insts: - assert isinstance(item, tuple) - inst = list(item) - opcode = dis.opmap[inst[0]] - oparg = inst[1] - loc = inst[2:] + [-1] * (6 - len(inst)) - res.append((opcode, oparg, *loc)) - return res + if isinstance(item, self.Label): + seq.use_label(item.value) + else: + op = item[0] + if isinstance(op, str): + op = opcode.opmap[op] + arg, *loc = item[1:] + if isinstance(arg, self.Label): + arg = arg.value + loc = loc + [-1] * (4 - len(loc)) + seq.addop(op, arg or 0, *loc) + return seq + + def check_instructions(self, insts): + for inst in insts: + if isinstance(inst, self.Label): + continue + op, arg, *loc = inst + if isinstance(op, str): + op = opcode.opmap[op] + self.assertEqual(op in opcode.hasarg, + arg is not None, + f"{opcode.opname[op]=} {arg=}") + self.assertTrue(all(isinstance(l, int) for l in loc)) @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") @@ -147,10 +146,8 @@ class CodegenTestCase(CompilationStepTestCase): @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") class CfgOptimizationTestCase(CompilationStepTestCase): - def get_optimized(self, insts, consts, nlocals=0): - insts = self.normalize_insts(insts) - insts = self.complete_insts_info(insts) - insts = _testinternalcapi.optimize_cfg(insts, consts, nlocals) + def get_optimized(self, seq, consts, nlocals=0): + insts = _testinternalcapi.optimize_cfg(seq, consts, nlocals) return insts, consts @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 638b6e96b50..484d72e6341 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -2,6 +2,7 @@ import contextlib import dis import io import math +import opcode import os import unittest import sys @@ -11,6 +12,8 @@ import tempfile import types import textwrap import warnings +import _testinternalcapi + from test import support from test.support import (script_helper, requires_debug_ranges, requires_specialization, get_c_recursion_limit) @@ -2419,6 +2422,49 @@ class TestStackSizeStability(unittest.TestCase): """ self.check_stack_size(snippet, async_=True) +class TestInstructionSequence(unittest.TestCase): + def compare_instructions(self, seq, expected): + self.assertEqual([(opcode.opname[i[0]],) + i[1:] for i in seq.get_instructions()], + expected) + + def test_basics(self): + seq = _testinternalcapi.new_instruction_sequence() + + def add_op(seq, opname, oparg, bl, bc=0, el=0, ec=0): + seq.addop(opcode.opmap[opname], oparg, bl, bc, el, el) + + add_op(seq, 'LOAD_CONST', 1, 1) + add_op(seq, 'JUMP', lbl1 := seq.new_label(), 2) + add_op(seq, 'LOAD_CONST', 1, 3) + add_op(seq, 'JUMP', lbl2 := seq.new_label(), 4) + seq.use_label(lbl1) + add_op(seq, 'LOAD_CONST', 2, 4) + seq.use_label(lbl2) + add_op(seq, 'RETURN_VALUE', 0, 3) + + expected = [('LOAD_CONST', 1, 1), + ('JUMP', 4, 2), + ('LOAD_CONST', 1, 3), + ('JUMP', 5, 4), + ('LOAD_CONST', 2, 4), + ('RETURN_VALUE', None, 3), + ] + + self.compare_instructions(seq, [ex + (0,0,0) for ex in expected]) + + def test_nested(self): + seq = _testinternalcapi.new_instruction_sequence() + seq.addop(opcode.opmap['LOAD_CONST'], 1, 1, 0, 0, 0) + nested = _testinternalcapi.new_instruction_sequence() + nested.addop(opcode.opmap['LOAD_CONST'], 2, 2, 0, 0, 0) + + self.compare_instructions(seq, [('LOAD_CONST', 1, 1, 0, 0, 0)]) + self.compare_instructions(nested, [('LOAD_CONST', 2, 2, 0, 0, 0)]) + + seq.add_nested(nested) + self.compare_instructions(seq, [('LOAD_CONST', 1, 1, 0, 0, 0)]) + self.compare_instructions(seq.get_nested()[0], [('LOAD_CONST', 2, 2, 0, 0, 0)]) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_compiler_assemble.py b/Lib/test/test_compiler_assemble.py index ab9f04dd63a..1b98b0d97ed 100644 --- a/Lib/test/test_compiler_assemble.py +++ b/Lib/test/test_compiler_assemble.py @@ -27,8 +27,8 @@ class IsolatedAssembleTests(AssemblerTestCase): def insts_to_code_object(self, insts, metadata): metadata = self.complete_metadata(metadata) - insts = self.complete_insts_info(insts) - return self.get_code_object(metadata['filename'], insts, metadata) + seq = self.seq_from_insts(insts) + return self.get_code_object(metadata['filename'], seq, metadata) def assemble_test(self, insts, metadata, expected): co = self.insts_to_code_object(insts, metadata) @@ -71,7 +71,7 @@ class IsolatedAssembleTests(AssemblerTestCase): ('BINARY_OP', 0, 1), # '+' ('LOAD_CONST', 0, 1), # 2 ('BINARY_OP', 11, 1), # '/' - ('RETURN_VALUE', 1), + ('RETURN_VALUE', None, 1), ] expected = {(3, 4) : 3.5, (-100, 200) : 50, (10, 18) : 14} self.assemble_test(insts, metadata, expected) @@ -102,13 +102,13 @@ class IsolatedAssembleTests(AssemblerTestCase): ('LOAD_CLOSURE', 0, 1), ('BUILD_TUPLE', 1, 1), ('LOAD_CONST', 1, 1), - ('MAKE_FUNCTION', 0, 2), + ('MAKE_FUNCTION', None, 2), ('SET_FUNCTION_ATTRIBUTE', 8, 2), - ('PUSH_NULL', 0, 1), + ('PUSH_NULL', None, 1), ('CALL', 0, 2), # (lambda: x)() ('LOAD_CONST', 2, 2), # 2 ('BINARY_OP', 6, 2), # % - ('RETURN_VALUE', 0, 2) + ('RETURN_VALUE', None, 2) ] expected = {(0,): 0, (1,): 1, (2,): 0, (120,): 0, (121,): 1} @@ -128,12 +128,12 @@ class IsolatedAssembleTests(AssemblerTestCase): ('SETUP_FINALLY', 3), ('RETURN_CONST', 0), ('SETUP_CLEANUP', 8), - ('PUSH_EXC_INFO', 0), - ('POP_TOP', 0), - ('POP_EXCEPT', 0), + ('PUSH_EXC_INFO', None), + ('POP_TOP', None), + ('POP_EXCEPT', None), ('RETURN_CONST', 0), ('COPY', 3), - ('POP_EXCEPT', 0), + ('POP_EXCEPT', None), ('RERAISE', 1), ] co = self.insts_to_code_object(insts, metadata) diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index dffedd0b1fc..6989aafd293 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -1,5 +1,6 @@ import dis from itertools import combinations, product +import opcode import sys import textwrap import unittest @@ -981,10 +982,15 @@ class DirectCfgOptimizerTests(CfgOptimizationTestCase): def cfg_optimization_test(self, insts, expected_insts, consts=None, expected_consts=None, nlocals=0): + + self.check_instructions(insts) + self.check_instructions(expected_insts) + if expected_consts is None: expected_consts = consts - opt_insts, opt_consts = self.get_optimized(insts, consts, nlocals) - expected_insts = self.normalize_insts(expected_insts) + seq = self.seq_from_insts(insts) + opt_insts, opt_consts = self.get_optimized(seq, consts, nlocals) + expected_insts = self.seq_from_insts(expected_insts).get_instructions() self.assertInstructionsMatch(opt_insts, expected_insts) self.assertEqual(opt_consts, expected_consts) @@ -993,10 +999,10 @@ class DirectCfgOptimizerTests(CfgOptimizationTestCase): ('LOAD_NAME', 1, 11), ('POP_JUMP_IF_TRUE', lbl := self.Label(), 12), ('LOAD_CONST', 2, 13), - ('RETURN_VALUE', 13), + ('RETURN_VALUE', None, 13), lbl, ('LOAD_CONST', 3, 14), - ('RETURN_VALUE', 14), + ('RETURN_VALUE', None, 14), ] expected_insts = [ ('LOAD_NAME', 1, 11), @@ -1020,11 +1026,11 @@ class DirectCfgOptimizerTests(CfgOptimizationTestCase): ('LOAD_CONST', 2, 13), lbl, ('LOAD_CONST', 3, 14), - ('RETURN_VALUE', 14), + ('RETURN_VALUE', None, 14), ] expected_insts = [ - ('NOP', 11), - ('NOP', 12), + ('NOP', None, 11), + ('NOP', None, 12), ('RETURN_CONST', 1, 14), ] self.cfg_optimization_test(insts, @@ -1038,14 +1044,14 @@ class DirectCfgOptimizerTests(CfgOptimizationTestCase): ('LOAD_NAME', 1, 11), ('POP_JUMP_IF_TRUE', lbl1, 12), ('LOAD_NAME', 2, 13), - ('RETURN_VALUE', 13), + ('RETURN_VALUE', None, 13), ] expected = [ lbl := self.Label(), ('LOAD_NAME', 1, 11), ('POP_JUMP_IF_TRUE', lbl, 12), ('LOAD_NAME', 2, 13), - ('RETURN_VALUE', 13), + ('RETURN_VALUE', None, 13), ] self.cfg_optimization_test(insts, expected, consts=list(range(5))) @@ -1056,11 +1062,11 @@ class DirectCfgOptimizerTests(CfgOptimizationTestCase): ('LOAD_CONST', 3, 11), ('POP_JUMP_IF_TRUE', lbl1, 12), ('LOAD_CONST', 2, 13), - ('RETURN_VALUE', 13), + ('RETURN_VALUE', None, 13), ] expected_insts = [ lbl := self.Label(), - ('NOP', 11), + ('NOP', None, 11), ('JUMP', lbl, 12), ] self.cfg_optimization_test(insts, expected_insts, consts=list(range(5))) @@ -1068,7 +1074,7 @@ class DirectCfgOptimizerTests(CfgOptimizationTestCase): def test_except_handler_label(self): insts = [ ('SETUP_FINALLY', handler := self.Label(), 10), - ('POP_BLOCK', 0, -1), + ('POP_BLOCK', None, -1), ('RETURN_CONST', 1, 11), handler, ('RETURN_CONST', 2, 12), @@ -1090,16 +1096,16 @@ class DirectCfgOptimizerTests(CfgOptimizationTestCase): ('SWAP', 3, 4), ('STORE_FAST', 1, 4), ('STORE_FAST', 1, 4), - ('POP_TOP', 0, 4), + ('POP_TOP', None, 4), ('LOAD_CONST', 0, 5), - ('RETURN_VALUE', 5) + ('RETURN_VALUE', None, 5) ] expected_insts = [ ('LOAD_CONST', 0, 1), ('LOAD_CONST', 1, 2), - ('NOP', 0, 3), + ('NOP', None, 3), ('STORE_FAST', 1, 4), - ('POP_TOP', 0, 4), + ('POP_TOP', None, 4), ('RETURN_CONST', 0) ] self.cfg_optimization_test(insts, expected_insts, consts=list(range(3)), nlocals=1) @@ -1113,13 +1119,13 @@ class DirectCfgOptimizerTests(CfgOptimizationTestCase): ('STORE_FAST', 1, 4), ('STORE_FAST', 1, 4), ('LOAD_CONST', 0, 5), - ('RETURN_VALUE', 5) + ('RETURN_VALUE', None, 5) ] expected_insts = [ ('LOAD_CONST', 0, 1), ('LOAD_CONST', 1, 2), - ('NOP', 0, 3), - ('POP_TOP', 0, 4), + ('NOP', None, 3), + ('POP_TOP', None, 4), ('STORE_FAST', 1, 4), ('RETURN_CONST', 0, 5) ] @@ -1134,7 +1140,7 @@ class DirectCfgOptimizerTests(CfgOptimizationTestCase): ('STORE_FAST', 1, 5), ('STORE_FAST', 1, 6), ('LOAD_CONST', 0, 5), - ('RETURN_VALUE', 5) + ('RETURN_VALUE', None, 5) ] expected_insts = [ ('LOAD_CONST', 0, 1), @@ -1169,7 +1175,7 @@ class DirectCfgOptimizerTests(CfgOptimizationTestCase): op = 'JUMP' if 'JUMP' in (op1, op2) else 'JUMP_NO_INTERRUPT' expected_insts = [ ('LOAD_NAME', 0, 10), - ('NOP', 0, 4), + ('NOP', None, 4), (op, 0, 5), ] self.cfg_optimization_test(insts, expected_insts, consts=list(range(5))) diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-09-16-07-00.gh-issue-117680.MRZ78K.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-09-16-07-00.gh-issue-117680.MRZ78K.rst new file mode 100644 index 00000000000..fd437b7fdd3 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-09-16-07-00.gh-issue-117680.MRZ78K.rst @@ -0,0 +1 @@ +Give ``_PyInstructionSequence`` a Python interface and use it in tests. diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 36dad024d31..cc9e1403f87 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -22,6 +22,7 @@ #include "pycore_gc.h" // PyGC_Head #include "pycore_hashtable.h" // _Py_hashtable_new() #include "pycore_initconfig.h" // _Py_GetConfigsAsDict() +#include "pycore_instruction_sequence.h" // _PyInstructionSequence_New() #include "pycore_interp.h" // _PyInterpreterState_GetConfigCopy() #include "pycore_long.h" // _PyLong_Sign() #include "pycore_object.h" // _PyObject_IsFreed() @@ -723,6 +724,19 @@ _testinternalcapi_compiler_cleandoc_impl(PyObject *module, PyObject *doc) return _PyCompile_CleanDoc(doc); } +/*[clinic input] + +_testinternalcapi.new_instruction_sequence -> object + +Return a new, empty InstructionSequence. +[clinic start generated code]*/ + +static PyObject * +_testinternalcapi_new_instruction_sequence_impl(PyObject *module) +/*[clinic end generated code: output=ea4243fddb9057fd input=1dec2591b173be83]*/ +{ + return _PyInstructionSequence_New(); +} /*[clinic input] @@ -1952,6 +1966,7 @@ static PyMethodDef module_functions[] = { {"set_eval_frame_default", set_eval_frame_default, METH_NOARGS, NULL}, {"set_eval_frame_record", set_eval_frame_record, METH_O, NULL}, _TESTINTERNALCAPI_COMPILER_CLEANDOC_METHODDEF + _TESTINTERNALCAPI_NEW_INSTRUCTION_SEQUENCE_METHODDEF _TESTINTERNALCAPI_COMPILER_CODEGEN_METHODDEF _TESTINTERNALCAPI_OPTIMIZE_CFG_METHODDEF _TESTINTERNALCAPI_ASSEMBLE_CODE_OBJECT_METHODDEF diff --git a/Modules/clinic/_testinternalcapi.c.h b/Modules/clinic/_testinternalcapi.c.h index cba2a943d03..a61858561d5 100644 --- a/Modules/clinic/_testinternalcapi.c.h +++ b/Modules/clinic/_testinternalcapi.c.h @@ -67,6 +67,24 @@ exit: return return_value; } +PyDoc_STRVAR(_testinternalcapi_new_instruction_sequence__doc__, +"new_instruction_sequence($module, /)\n" +"--\n" +"\n" +"Return a new, empty InstructionSequence."); + +#define _TESTINTERNALCAPI_NEW_INSTRUCTION_SEQUENCE_METHODDEF \ + {"new_instruction_sequence", (PyCFunction)_testinternalcapi_new_instruction_sequence, METH_NOARGS, _testinternalcapi_new_instruction_sequence__doc__}, + +static PyObject * +_testinternalcapi_new_instruction_sequence_impl(PyObject *module); + +static PyObject * +_testinternalcapi_new_instruction_sequence(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testinternalcapi_new_instruction_sequence_impl(module); +} + PyDoc_STRVAR(_testinternalcapi_compiler_codegen__doc__, "compiler_codegen($module, /, ast, filename, optimize, compile_mode=0)\n" "--\n" @@ -282,4 +300,4 @@ _testinternalcapi_test_long_numbits(PyObject *module, PyObject *Py_UNUSED(ignore { return _testinternalcapi_test_long_numbits_impl(module); } -/*[clinic end generated code: output=679bf53bbae20085 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9804015d77d36c77 input=a9049054013a1b77]*/ diff --git a/Objects/object.c b/Objects/object.c index 3830db0650b..214e7c5b567 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -10,6 +10,7 @@ #include "pycore_dict.h" // _PyObject_MakeDictFromInstanceAttributes() #include "pycore_floatobject.h" // _PyFloat_DebugMallocStats() #include "pycore_initconfig.h" // _PyStatus_EXCEPTION() +#include "pycore_instruction_sequence.h" // _PyInstructionSequence_Type #include "pycore_hashtable.h" // _Py_hashtable_new() #include "pycore_memoryobject.h" // _PyManagedBuffer_Type #include "pycore_namespace.h" // _PyNamespace_Type @@ -2294,6 +2295,7 @@ static PyTypeObject* static_types[] = { &_PyHamt_BitmapNode_Type, &_PyHamt_CollisionNode_Type, &_PyHamt_Type, + &_PyInstructionSequence_Type, &_PyLegacyEventHandler_Type, &_PyLineIterator, &_PyManagedBuffer_Type, diff --git a/Python/clinic/instruction_sequence.c.h b/Python/clinic/instruction_sequence.c.h new file mode 100644 index 00000000000..66c2ccadfa0 --- /dev/null +++ b/Python/clinic/instruction_sequence.c.h @@ -0,0 +1,304 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif +#include "pycore_modsupport.h" // _PyArg_NoKeywords() + +PyDoc_STRVAR(inst_seq_new__doc__, +"InstructionSequenceType()\n" +"--\n" +"\n" +"Create a new InstructionSequence object."); + +static PyObject * +inst_seq_new_impl(PyTypeObject *type); + +static PyObject * +inst_seq_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + PyTypeObject *base_tp = &_PyInstructionSequence_Type; + + if ((type == base_tp || type->tp_init == base_tp->tp_init) && + !_PyArg_NoPositional("InstructionSequenceType", args)) { + goto exit; + } + if ((type == base_tp || type->tp_init == base_tp->tp_init) && + !_PyArg_NoKeywords("InstructionSequenceType", kwargs)) { + goto exit; + } + return_value = inst_seq_new_impl(type); + +exit: + return return_value; +} + +PyDoc_STRVAR(InstructionSequenceType_use_label__doc__, +"use_label($self, /, label)\n" +"--\n" +"\n" +"Place label at current location."); + +#define INSTRUCTIONSEQUENCETYPE_USE_LABEL_METHODDEF \ + {"use_label", _PyCFunction_CAST(InstructionSequenceType_use_label), METH_FASTCALL|METH_KEYWORDS, InstructionSequenceType_use_label__doc__}, + +static PyObject * +InstructionSequenceType_use_label_impl(_PyInstructionSequence *self, + int label); + +static PyObject * +InstructionSequenceType_use_label(_PyInstructionSequence *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(label), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"label", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "use_label", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + int label; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + label = PyLong_AsInt(args[0]); + if (label == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = InstructionSequenceType_use_label_impl(self, label); + +exit: + return return_value; +} + +PyDoc_STRVAR(InstructionSequenceType_addop__doc__, +"addop($self, /, opcode, oparg, lineno, col_offset, end_lineno,\n" +" end_col_offset)\n" +"--\n" +"\n" +"Append an instruction."); + +#define INSTRUCTIONSEQUENCETYPE_ADDOP_METHODDEF \ + {"addop", _PyCFunction_CAST(InstructionSequenceType_addop), METH_FASTCALL|METH_KEYWORDS, InstructionSequenceType_addop__doc__}, + +static PyObject * +InstructionSequenceType_addop_impl(_PyInstructionSequence *self, int opcode, + int oparg, int lineno, int col_offset, + int end_lineno, int end_col_offset); + +static PyObject * +InstructionSequenceType_addop(_PyInstructionSequence *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 6 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(opcode), &_Py_ID(oparg), &_Py_ID(lineno), &_Py_ID(col_offset), &_Py_ID(end_lineno), &_Py_ID(end_col_offset), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"opcode", "oparg", "lineno", "col_offset", "end_lineno", "end_col_offset", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "addop", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[6]; + int opcode; + int oparg; + int lineno; + int col_offset; + int end_lineno; + int end_col_offset; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 6, 6, 0, argsbuf); + if (!args) { + goto exit; + } + opcode = PyLong_AsInt(args[0]); + if (opcode == -1 && PyErr_Occurred()) { + goto exit; + } + oparg = PyLong_AsInt(args[1]); + if (oparg == -1 && PyErr_Occurred()) { + goto exit; + } + lineno = PyLong_AsInt(args[2]); + if (lineno == -1 && PyErr_Occurred()) { + goto exit; + } + col_offset = PyLong_AsInt(args[3]); + if (col_offset == -1 && PyErr_Occurred()) { + goto exit; + } + end_lineno = PyLong_AsInt(args[4]); + if (end_lineno == -1 && PyErr_Occurred()) { + goto exit; + } + end_col_offset = PyLong_AsInt(args[5]); + if (end_col_offset == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = InstructionSequenceType_addop_impl(self, opcode, oparg, lineno, col_offset, end_lineno, end_col_offset); + +exit: + return return_value; +} + +PyDoc_STRVAR(InstructionSequenceType_new_label__doc__, +"new_label($self, /)\n" +"--\n" +"\n" +"Return a new label."); + +#define INSTRUCTIONSEQUENCETYPE_NEW_LABEL_METHODDEF \ + {"new_label", (PyCFunction)InstructionSequenceType_new_label, METH_NOARGS, InstructionSequenceType_new_label__doc__}, + +static int +InstructionSequenceType_new_label_impl(_PyInstructionSequence *self); + +static PyObject * +InstructionSequenceType_new_label(_PyInstructionSequence *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + int _return_value; + + _return_value = InstructionSequenceType_new_label_impl(self); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromLong((long)_return_value); + +exit: + return return_value; +} + +PyDoc_STRVAR(InstructionSequenceType_add_nested__doc__, +"add_nested($self, /, nested)\n" +"--\n" +"\n" +"Add a nested sequence."); + +#define INSTRUCTIONSEQUENCETYPE_ADD_NESTED_METHODDEF \ + {"add_nested", _PyCFunction_CAST(InstructionSequenceType_add_nested), METH_FASTCALL|METH_KEYWORDS, InstructionSequenceType_add_nested__doc__}, + +static PyObject * +InstructionSequenceType_add_nested_impl(_PyInstructionSequence *self, + PyObject *nested); + +static PyObject * +InstructionSequenceType_add_nested(_PyInstructionSequence *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(nested), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"nested", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "add_nested", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *nested; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + nested = args[0]; + return_value = InstructionSequenceType_add_nested_impl(self, nested); + +exit: + return return_value; +} + +PyDoc_STRVAR(InstructionSequenceType_get_nested__doc__, +"get_nested($self, /)\n" +"--\n" +"\n" +"Add a nested sequence."); + +#define INSTRUCTIONSEQUENCETYPE_GET_NESTED_METHODDEF \ + {"get_nested", (PyCFunction)InstructionSequenceType_get_nested, METH_NOARGS, InstructionSequenceType_get_nested__doc__}, + +static PyObject * +InstructionSequenceType_get_nested_impl(_PyInstructionSequence *self); + +static PyObject * +InstructionSequenceType_get_nested(_PyInstructionSequence *self, PyObject *Py_UNUSED(ignored)) +{ + return InstructionSequenceType_get_nested_impl(self); +} + +PyDoc_STRVAR(InstructionSequenceType_get_instructions__doc__, +"get_instructions($self, /)\n" +"--\n" +"\n" +"Return the instructions as a list of tuples or labels."); + +#define INSTRUCTIONSEQUENCETYPE_GET_INSTRUCTIONS_METHODDEF \ + {"get_instructions", (PyCFunction)InstructionSequenceType_get_instructions, METH_NOARGS, InstructionSequenceType_get_instructions__doc__}, + +static PyObject * +InstructionSequenceType_get_instructions_impl(_PyInstructionSequence *self); + +static PyObject * +InstructionSequenceType_get_instructions(_PyInstructionSequence *self, PyObject *Py_UNUSED(ignored)) +{ + return InstructionSequenceType_get_instructions_impl(self); +} +/*[clinic end generated code: output=8809d7aa11d9b2bb input=a9049054013a1b77]*/ diff --git a/Python/compile.c b/Python/compile.c index 1e8f97e72cd..3d856b7e4dd 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -32,6 +32,7 @@ #include "pycore_code.h" // _PyCode_New() #include "pycore_compile.h" #include "pycore_flowgraph.h" +#include "pycore_instruction_sequence.h" // _PyInstructionSequence_New() #include "pycore_intrinsics.h" #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_pystate.h" // _Py_GetConfig() @@ -248,7 +249,7 @@ struct compiler_unit { PyObject *u_private; /* for private name mangling */ PyObject *u_static_attributes; /* for class: attributes accessed via self.X */ - instr_sequence u_instr_sequence; /* codegen output */ + instr_sequence *u_instr_sequence; /* codegen output */ int u_nfblocks; int u_in_inlined_comp; @@ -281,12 +282,12 @@ struct compiler { int c_nestlevel; PyObject *c_const_cache; /* Python dict holding all constants, including names tuple */ - struct compiler_unit *u; /* compiler state for current block */ + struct compiler_unit *u; /* compiler state for current block */ PyObject *c_stack; /* Python list holding compiler_unit ptrs */ PyArena *c_arena; /* pointer to memory allocation arena */ }; -#define INSTR_SEQUENCE(C) (&((C)->u->u_instr_sequence)) +#define INSTR_SEQUENCE(C) ((C)->u->u_instr_sequence) typedef struct { @@ -567,7 +568,7 @@ dictbytype(PyObject *src, int scope_type, int flag, Py_ssize_t offset) static void compiler_unit_free(struct compiler_unit *u) { - PyInstructionSequence_Fini(&u->u_instr_sequence); + Py_CLEAR(u->u_instr_sequence); Py_CLEAR(u->u_ste); Py_CLEAR(u->u_metadata.u_name); Py_CLEAR(u->u_metadata.u_qualname); @@ -976,7 +977,7 @@ compiler_addop_load_const(PyObject *const_cache, struct compiler_unit *u, locati if (arg < 0) { return ERROR; } - return codegen_addop_i(&u->u_instr_sequence, LOAD_CONST, arg, loc); + return codegen_addop_i(u->u_instr_sequence, LOAD_CONST, arg, loc); } static int @@ -987,7 +988,7 @@ compiler_addop_o(struct compiler_unit *u, location loc, if (arg < 0) { return ERROR; } - return codegen_addop_i(&u->u_instr_sequence, opcode, arg, loc); + return codegen_addop_i(u->u_instr_sequence, opcode, arg, loc); } static int @@ -1033,7 +1034,7 @@ compiler_addop_name(struct compiler_unit *u, location loc, arg <<= 2; arg |= 1; } - return codegen_addop_i(&u->u_instr_sequence, opcode, arg, loc); + return codegen_addop_i(u->u_instr_sequence, opcode, arg, loc); } /* Add an opcode with an integer argument */ @@ -1252,6 +1253,8 @@ compiler_enter_scope(struct compiler *c, identifier name, u->u_static_attributes = NULL; } + u->u_instr_sequence = (instr_sequence*)_PyInstructionSequence_New(); + /* Push the old compiler_unit on the stack. */ if (c->u) { PyObject *capsule = PyCapsule_New(c->u, CAPSULE_NAME, NULL); @@ -7526,7 +7529,7 @@ optimize_and_assemble_code_unit(struct compiler_unit *u, PyObject *const_cache, if (consts == NULL) { goto error; } - g = instr_sequence_to_cfg(&u->u_instr_sequence); + g = instr_sequence_to_cfg(u->u_instr_sequence); if (g == NULL) { goto error; } @@ -7593,160 +7596,25 @@ optimize_and_assemble(struct compiler *c, int addNone) * a jump target label marking the beginning of a basic block. */ -static int -instructions_to_instr_sequence(PyObject *instructions, instr_sequence *seq) -{ - assert(PyList_Check(instructions)); - - Py_ssize_t num_insts = PyList_GET_SIZE(instructions); - bool *is_target = PyMem_Calloc(num_insts, sizeof(bool)); - if (is_target == NULL) { - return ERROR; - } - for (Py_ssize_t i = 0; i < num_insts; i++) { - PyObject *item = PyList_GET_ITEM(instructions, i); - if (!PyTuple_Check(item) || PyTuple_GET_SIZE(item) != 6) { - PyErr_SetString(PyExc_ValueError, "expected a 6-tuple"); - goto error; - } - int opcode = PyLong_AsLong(PyTuple_GET_ITEM(item, 0)); - if (PyErr_Occurred()) { - goto error; - } - if (HAS_TARGET(opcode)) { - int oparg = PyLong_AsLong(PyTuple_GET_ITEM(item, 1)); - if (PyErr_Occurred()) { - goto error; - } - if (oparg < 0 || oparg >= num_insts) { - PyErr_SetString(PyExc_ValueError, "label out of range"); - goto error; - } - is_target[oparg] = true; - } - } - - for (int i = 0; i < num_insts; i++) { - if (is_target[i]) { - if (_PyInstructionSequence_UseLabel(seq, i) < 0) { - goto error; - } - } - PyObject *item = PyList_GET_ITEM(instructions, i); - if (!PyTuple_Check(item) || PyTuple_GET_SIZE(item) != 6) { - PyErr_SetString(PyExc_ValueError, "expected a 6-tuple"); - goto error; - } - int opcode = PyLong_AsLong(PyTuple_GET_ITEM(item, 0)); - if (PyErr_Occurred()) { - goto error; - } - int oparg; - if (OPCODE_HAS_ARG(opcode)) { - oparg = PyLong_AsLong(PyTuple_GET_ITEM(item, 1)); - if (PyErr_Occurred()) { - goto error; - } - } - else { - oparg = 0; - } - location loc; - loc.lineno = PyLong_AsLong(PyTuple_GET_ITEM(item, 2)); - if (PyErr_Occurred()) { - goto error; - } - loc.end_lineno = PyLong_AsLong(PyTuple_GET_ITEM(item, 3)); - if (PyErr_Occurred()) { - goto error; - } - loc.col_offset = PyLong_AsLong(PyTuple_GET_ITEM(item, 4)); - if (PyErr_Occurred()) { - goto error; - } - loc.end_col_offset = PyLong_AsLong(PyTuple_GET_ITEM(item, 5)); - if (PyErr_Occurred()) { - goto error; - } - if (_PyInstructionSequence_Addop(seq, opcode, oparg, loc) < 0) { - goto error; - } - } - PyMem_Free(is_target); - return SUCCESS; -error: - PyMem_Free(is_target); - return ERROR; -} - -static cfg_builder* -instructions_to_cfg(PyObject *instructions) -{ - cfg_builder *g = NULL; - instr_sequence seq; - memset(&seq, 0, sizeof(instr_sequence)); - - if (instructions_to_instr_sequence(instructions, &seq) < 0) { - goto error; - } - g = instr_sequence_to_cfg(&seq); - if (g == NULL) { - goto error; - } - PyInstructionSequence_Fini(&seq); - return g; -error: - _PyCfgBuilder_Free(g); - PyInstructionSequence_Fini(&seq); - return NULL; -} static PyObject * -instr_sequence_to_instructions(instr_sequence *seq) +cfg_to_instruction_sequence(cfg_builder *g) { - PyObject *instructions = PyList_New(0); - if (instructions == NULL) { - return NULL; - } - for (int i = 0; i < seq->s_used; i++) { - instruction *instr = &seq->s_instrs[i]; - location loc = instr->i_loc; - PyObject *inst_tuple = Py_BuildValue( - "(iiiiii)", instr->i_opcode, instr->i_oparg, - loc.lineno, loc.end_lineno, - loc.col_offset, loc.end_col_offset); - if (inst_tuple == NULL) { + instr_sequence *seq = (instr_sequence *)_PyInstructionSequence_New(); + if (seq != NULL) { + if (_PyCfg_ToInstructionSequence(g, seq) < 0) { goto error; } - - int res = PyList_Append(instructions, inst_tuple); - Py_DECREF(inst_tuple); - if (res != 0) { + if (_PyInstructionSequence_ApplyLabelMap(seq) < 0) { goto error; } } - return instructions; + return (PyObject*)seq; error: - Py_XDECREF(instructions); + PyInstructionSequence_Fini(seq); return NULL; } -static PyObject * -cfg_to_instructions(cfg_builder *g) -{ - instr_sequence seq; - memset(&seq, 0, sizeof(seq)); - if (_PyCfg_ToInstructionSequence(g, &seq) < 0) { - return NULL; - } - if (_PyInstructionSequence_ApplyLabelMap(&seq) < 0) { - return NULL; - } - PyObject *res = instr_sequence_to_instructions(&seq); - PyInstructionSequence_Fini(&seq); - return res; -} - // C implementation of inspect.cleandoc() // // Difference from inspect.cleandoc(): @@ -7916,13 +7784,8 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, if (_PyInstructionSequence_ApplyLabelMap(INSTR_SEQUENCE(c)) < 0) { return NULL; } - - PyObject *insts = instr_sequence_to_instructions(INSTR_SEQUENCE(c)); - if (insts == NULL) { - goto finally; - } - res = PyTuple_Pack(2, insts, metadata); - Py_DECREF(insts); + /* Allocate a copy of the instruction sequence on the heap */ + res = PyTuple_Pack(2, INSTR_SEQUENCE(c), metadata); finally: Py_XDECREF(metadata); @@ -7933,16 +7796,19 @@ finally: } PyObject * -_PyCompile_OptimizeCfg(PyObject *instructions, PyObject *consts, int nlocals) +_PyCompile_OptimizeCfg(PyObject *seq, PyObject *consts, int nlocals) { - cfg_builder *g = NULL; - PyObject *res = NULL; + if (!_PyInstructionSequence_Check(seq)) { + PyErr_SetString(PyExc_ValueError, "expected an instruction sequence"); + return NULL; + } PyObject *const_cache = PyDict_New(); if (const_cache == NULL) { return NULL; } - g = instructions_to_cfg(instructions); + PyObject *res = NULL; + cfg_builder *g = instr_sequence_to_cfg((instr_sequence*)seq); if (g == NULL) { goto error; } @@ -7951,7 +7817,7 @@ _PyCompile_OptimizeCfg(PyObject *instructions, PyObject *consts, int nlocals) nparams, firstlineno) < 0) { goto error; } - res = cfg_to_instructions(g); + res = cfg_to_instruction_sequence(g); error: Py_DECREF(const_cache); _PyCfgBuilder_Free(g); @@ -7962,8 +7828,12 @@ int _PyCfg_JumpLabelsToTargets(cfg_builder *g); PyCodeObject * _PyCompile_Assemble(_PyCompile_CodeUnitMetadata *umd, PyObject *filename, - PyObject *instructions) + PyObject *seq) { + if (!_PyInstructionSequence_Check(seq)) { + PyErr_SetString(PyExc_TypeError, "expected an instruction sequence"); + return NULL; + } cfg_builder *g = NULL; PyCodeObject *co = NULL; instr_sequence optimized_instrs; @@ -7974,7 +7844,7 @@ _PyCompile_Assemble(_PyCompile_CodeUnitMetadata *umd, PyObject *filename, return NULL; } - g = instructions_to_cfg(instructions); + g = instr_sequence_to_cfg((instr_sequence*)seq); if (g == NULL) { goto error; } diff --git a/Python/instruction_sequence.c b/Python/instruction_sequence.c index 597d2b73d19..d843d5f1690 100644 --- a/Python/instruction_sequence.c +++ b/Python/instruction_sequence.c @@ -141,6 +141,21 @@ _PyInstructionSequence_InsertInstruction(instr_sequence *seq, int pos, return SUCCESS; } +int +_PyInstructionSequence_AddNested(instr_sequence *seq, instr_sequence *nested) +{ + if (seq->s_nested == NULL) { + seq->s_nested = PyList_New(0); + if (seq->s_nested == NULL) { + return ERROR; + } + } + if (PyList_Append(seq->s_nested, (PyObject*)nested) < 0) { + return ERROR; + } + return SUCCESS; +} + void PyInstructionSequence_Fini(instr_sequence *seq) { PyMem_Free(seq->s_labelmap); @@ -149,3 +164,288 @@ PyInstructionSequence_Fini(instr_sequence *seq) { PyMem_Free(seq->s_instrs); seq->s_instrs = NULL; } + +/*[clinic input] +class InstructionSequenceType "_PyInstructionSequence *" "&_PyInstructionSequence_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=589963e07480390f]*/ + +#include "clinic/instruction_sequence.c.h" + +static _PyInstructionSequence* +inst_seq_create(void) +{ + _PyInstructionSequence *seq; + seq = PyObject_GC_New(_PyInstructionSequence, &_PyInstructionSequence_Type); + if (seq == NULL) { + return NULL; + } + seq->s_instrs = NULL; + seq->s_allocated = 0; + seq->s_used = 0; + seq->s_next_free_label = 0; + seq->s_labelmap = NULL; + seq->s_labelmap_size = 0; + seq->s_nested = NULL; + + PyObject_GC_Track(seq); + return seq; +} + +PyObject* +_PyInstructionSequence_New(void) +{ + _PyInstructionSequence *seq = inst_seq_create(); + if (seq == NULL) { + return NULL; + } + return (PyObject*)seq; +} + +/*[clinic input] +@classmethod +InstructionSequenceType.__new__ as inst_seq_new + +Create a new InstructionSequence object. +[clinic start generated code]*/ + +static PyObject * +inst_seq_new_impl(PyTypeObject *type) +/*[clinic end generated code: output=98881de92c8876f6 input=b393150146849c74]*/ +{ + return (PyObject*)inst_seq_create(); +} + +/*[clinic input] +InstructionSequenceType.use_label + + label: int + +Place label at current location. +[clinic start generated code]*/ + +static PyObject * +InstructionSequenceType_use_label_impl(_PyInstructionSequence *self, + int label) +/*[clinic end generated code: output=4c06bbacb2854755 input=da55f49bb91841f3]*/ + +{ + if (_PyInstructionSequence_UseLabel(self, label) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +/*[clinic input] +InstructionSequenceType.addop + + opcode: int + oparg: int + lineno: int + col_offset: int + end_lineno: int + end_col_offset: int + +Append an instruction. +[clinic start generated code]*/ + +static PyObject * +InstructionSequenceType_addop_impl(_PyInstructionSequence *self, int opcode, + int oparg, int lineno, int col_offset, + int end_lineno, int end_col_offset) +/*[clinic end generated code: output=af0cc22c048dfbf3 input=012762ac88198713]*/ +{ + _Py_SourceLocation loc = {lineno, col_offset, end_lineno, end_col_offset}; + if (_PyInstructionSequence_Addop(self, opcode, oparg, loc) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +/*[clinic input] +InstructionSequenceType.new_label -> int + +Return a new label. +[clinic start generated code]*/ + +static int +InstructionSequenceType_new_label_impl(_PyInstructionSequence *self) +/*[clinic end generated code: output=dcb0589e4f5bf4bd input=c66040b9897bc327]*/ +{ + _PyJumpTargetLabel lbl = _PyInstructionSequence_NewLabel(self); + return lbl.id; +} + +/*[clinic input] +InstructionSequenceType.add_nested + + nested: object + +Add a nested sequence. +[clinic start generated code]*/ + +static PyObject * +InstructionSequenceType_add_nested_impl(_PyInstructionSequence *self, + PyObject *nested) +/*[clinic end generated code: output=14540fad459f7971 input=f2c482568b3b3c0f]*/ +{ + if (!_PyInstructionSequence_Check(nested)) { + PyErr_Format(PyExc_TypeError, + "expected an instruction sequence, not %T", + Py_TYPE(nested)); + return NULL; + } + if (_PyInstructionSequence_AddNested(self, (_PyInstructionSequence*)nested) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +/*[clinic input] +InstructionSequenceType.get_nested + +Add a nested sequence. +[clinic start generated code]*/ + +static PyObject * +InstructionSequenceType_get_nested_impl(_PyInstructionSequence *self) +/*[clinic end generated code: output=f415112c292630cb input=e429e474c57b95b4]*/ +{ + if (self->s_nested == NULL) { + return PyList_New(0); + } + return Py_NewRef(self->s_nested); +} + +/*[clinic input] +InstructionSequenceType.get_instructions + +Return the instructions as a list of tuples or labels. +[clinic start generated code]*/ + +static PyObject * +InstructionSequenceType_get_instructions_impl(_PyInstructionSequence *self) +/*[clinic end generated code: output=23f4f3f894c301b3 input=fbadb5dadb611291]*/ +{ + if (_PyInstructionSequence_ApplyLabelMap(self) < 0) { + return NULL; + } + PyObject *instructions = PyList_New(0); + if (instructions == NULL) { + return NULL; + } + for (int i = 0; i < self->s_used; i++) { + instruction *instr = &self->s_instrs[i]; + location loc = instr->i_loc; + PyObject *inst_tuple; + + if (OPCODE_HAS_ARG(instr->i_opcode)) { + inst_tuple = Py_BuildValue( + "(iiiiii)", instr->i_opcode, instr->i_oparg, + loc.lineno, loc.end_lineno, + loc.col_offset, loc.end_col_offset); + } + else { + inst_tuple = Py_BuildValue( + "(iOiiii)", instr->i_opcode, Py_None, + loc.lineno, loc.end_lineno, + loc.col_offset, loc.end_col_offset); + } + if (inst_tuple == NULL) { + goto error; + } + + int res = PyList_Append(instructions, inst_tuple); + Py_DECREF(inst_tuple); + if (res != 0) { + goto error; + } + } + return instructions; +error: + Py_XDECREF(instructions); + return NULL; +} + +static PyMethodDef inst_seq_methods[] = { + INSTRUCTIONSEQUENCETYPE_ADDOP_METHODDEF + INSTRUCTIONSEQUENCETYPE_NEW_LABEL_METHODDEF + INSTRUCTIONSEQUENCETYPE_USE_LABEL_METHODDEF + INSTRUCTIONSEQUENCETYPE_ADD_NESTED_METHODDEF + INSTRUCTIONSEQUENCETYPE_GET_NESTED_METHODDEF + INSTRUCTIONSEQUENCETYPE_GET_INSTRUCTIONS_METHODDEF + {NULL, NULL, 0, NULL}, +}; + +static PyMemberDef inst_seq_memberlist[] = { + {NULL} /* Sentinel */ +}; + +static PyGetSetDef inst_seq_getsetters[] = { + {NULL} /* Sentinel */ +}; + +static void +inst_seq_dealloc(_PyInstructionSequence *seq) +{ + PyObject_GC_UnTrack(seq); + Py_TRASHCAN_BEGIN(seq, inst_seq_dealloc) + PyInstructionSequence_Fini(seq); + PyObject_GC_Del(seq); + Py_TRASHCAN_END +} + +static int +inst_seq_traverse(_PyInstructionSequence *seq, visitproc visit, void *arg) +{ + Py_VISIT(seq->s_nested); + return 0; +} + +static int +inst_seq_clear(_PyInstructionSequence *seq) +{ + Py_CLEAR(seq->s_nested); + return 0; +} + +PyTypeObject _PyInstructionSequence_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "InstructionSequence", + sizeof(_PyInstructionSequence), + 0, + (destructor)inst_seq_dealloc, /*tp_dealloc*/ + 0, /*tp_vectorcall_offset*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_as_async*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */ + inst_seq_new__doc__, /* tp_doc */ + (traverseproc)inst_seq_traverse, /* tp_traverse */ + (inquiry)inst_seq_clear, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + inst_seq_methods, /* tp_methods */ + inst_seq_memberlist, /* tp_members */ + inst_seq_getsetters, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + inst_seq_new, /* tp_new */ +}; diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 65f94e50e1b..79a8f850ec1 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -103,6 +103,7 @@ Python/bltinmodule.c - PyZip_Type - Python/context.c - PyContextToken_Type - Python/context.c - PyContextVar_Type - Python/context.c - PyContext_Type - +Python/instruction_sequence.c - _PyInstructionSequence_Type - Python/traceback.c - PyTraceBack_Type - ##-----------------------