Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

GH-111485: Factor out tier 2 code generation from the rest of the interpreter code generator (GH-112968)

This commit is contained in:
Mark Shannon 2023-12-12 12:12:17 +00:00 committed by GitHub
parent c454e934d3
commit 0c55f27060
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 1391 additions and 974 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Include/internal/pycore_uop_ids.h generated
View File

@ -1,6 +1,6 @@
// This file is generated by Tools/cases_generator/uop_id_generator.py
// from:
// ['./Python/bytecodes.c']
// Python/bytecodes.c
// Do not edit!
#ifndef Py_CORE_UOP_IDS_H
#define Py_CORE_UOP_IDS_H

2
Include/opcode_ids.h generated
View File

@ -1,6 +1,6 @@
// This file is generated by Tools/cases_generator/opcode_id_generator.py
// from:
// ['./Python/bytecodes.c']
// Python/bytecodes.c
// Do not edit!
#ifndef Py_OPCODE_IDS_H

3
Makefile.pre.in
View File

@ -1589,7 +1589,6 @@ regen-cases:
$(CASESFLAG) \
-t $(srcdir)/Python/opcode_targets.h.new \
-m $(srcdir)/Include/internal/pycore_opcode_metadata.h.new \
-e $(srcdir)/Python/executor_cases.c.h.new \
-p $(srcdir)/Lib/_opcode_metadata.py.new \
-a $(srcdir)/Python/abstract_interp_cases.c.h.new \
$(srcdir)/Python/bytecodes.c
@ -1599,6 +1598,8 @@ regen-cases:
$(srcdir)/Tools/cases_generator/uop_id_generator.py -o $(srcdir)/Include/internal/pycore_uop_ids.h.new $(srcdir)/Python/bytecodes.c
$(PYTHON_FOR_REGEN) \
$(srcdir)/Tools/cases_generator/tier1_generator.py -o $(srcdir)/Python/generated_cases.c.h.new $(srcdir)/Python/bytecodes.c
$(PYTHON_FOR_REGEN) \
$(srcdir)/Tools/cases_generator/tier2_generator.py -o $(srcdir)/Python/executor_cases.c.h.new $(srcdir)/Python/bytecodes.c
$(UPDATE_FILE) $(srcdir)/Python/generated_cases.c.h $(srcdir)/Python/generated_cases.c.h.new
$(UPDATE_FILE) $(srcdir)/Include/opcode_ids.h $(srcdir)/Include/opcode_ids.h.new
$(UPDATE_FILE) $(srcdir)/Include/internal/pycore_uop_ids.h $(srcdir)/Include/internal/pycore_uop_ids.h.new

3
Python/bytecodes.c
View File

@ -3967,6 +3967,7 @@ dummy_func(
}
inst(EXTENDED_ARG, ( -- )) {
TIER_ONE_ONLY
assert(oparg);
opcode = next_instr->op.code;
oparg = oparg << 8 | next_instr->op.arg;
@ -3975,11 +3976,13 @@ dummy_func(
}
inst(CACHE, (--)) {
TIER_ONE_ONLY
assert(0 && "Executing a cache.");
Py_UNREACHABLE();
}
inst(RESERVED, (--)) {
TIER_ONE_ONLY
assert(0 && "Executing RESERVED instruction.");
Py_UNREACHABLE();
}

1616
Python/executor_cases.c.h generated
View File

File diff suppressed because it is too large Load Diff

5
Python/generated_cases.c.h generated
View File

@ -1,6 +1,6 @@
// This file is generated by Tools/cases_generator/tier1_generator.py
// from:
// ['./Python/bytecodes.c']
// Python/bytecodes.c
// Do not edit!
#ifdef TIER_TWO
@ -725,6 +725,7 @@
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(CACHE);
TIER_ONE_ONLY
assert(0 && "Executing a cache.");
Py_UNREACHABLE();
}
@ -2364,6 +2365,7 @@
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(EXTENDED_ARG);
TIER_ONE_ONLY
assert(oparg);
opcode = next_instr->op.code;
oparg = oparg << 8 | next_instr->op.arg;
@ -4704,6 +4706,7 @@
frame->instr_ptr = next_instr;
next_instr += 1;
INSTRUCTION_STATS(RESERVED);
TIER_ONE_ONLY
assert(0 && "Executing RESERVED instruction.");
Py_UNREACHABLE();
}

19
Tools/cases_generator/analyzer.py
View File

@ -15,6 +15,7 @@ class Properties:
needs_this: bool
always_exits: bool
stores_sp: bool
tier_one_only: bool
def dump(self, indent: str) -> None:
print(indent, end="")
@ -33,6 +34,7 @@ class Properties:
needs_this=any(p.needs_this for p in properties),
always_exits=any(p.always_exits for p in properties),
stores_sp=any(p.stores_sp for p in properties),
tier_one_only=any(p.tier_one_only for p in properties),
)
@ -46,6 +48,7 @@ SKIP_PROPERTIES = Properties(
needs_this=False,
always_exits=False,
stores_sp=False,
tier_one_only=False,
)
@ -124,6 +127,21 @@ class Uop:
self._size = sum(c.size for c in self.caches)
return self._size
def is_viable(self) -> bool:
if self.name == "_SAVE_RETURN_OFFSET":
return True # Adjusts next_instr, but only in tier 1 code
if self.properties.needs_this:
return False
if "INSTRUMENTED" in self.name:
return False
if "replaced" in self.annotations:
return False
if self.name in ("INTERPRETER_EXIT", "JUMP_BACKWARD"):
return False
if len([c for c in self.caches if c.name != "unused"]) > 1:
return False
return True
Part = Uop | Skip
@ -292,6 +310,7 @@ def compute_properties(op: parser.InstDef) -> Properties:
needs_this=variable_used(op, "this_instr"),
always_exits=always_exits(op),
stores_sp=variable_used(op, "STORE_SP"),
tier_one_only=variable_used(op, "TIER_ONE_ONLY"),
)

8
Tools/cases_generator/generate_cases.py
View File

@ -128,13 +128,6 @@ arg_parser.add_argument(
arg_parser.add_argument(
"input", nargs=argparse.REMAINDER, help="Instruction definition file(s)"
)
arg_parser.add_argument(
"-e",
"--executor-cases",
type=str,
help="Write executor cases to this file",
default=DEFAULT_EXECUTOR_OUTPUT,
)
arg_parser.add_argument(
"-a",
"--abstract-interpreter-cases",
@ -846,7 +839,6 @@ def main() -> None:
a.assign_opcode_ids()
a.write_opcode_targets(args.opcode_targets_h)
a.write_metadata(args.metadata, args.pymetadata)
a.write_executor_instructions(args.executor_cases, args.emit_line_directives)
a.write_abstract_interpreter_instructions(
args.abstract_interpreter_cases, args.emit_line_directives
)

175
Tools/cases_generator/generators_common.py
View File

@ -1,19 +1,186 @@
from pathlib import Path
from typing import TextIO
from analyzer import (
Analysis,
Instruction,
Uop,
Part,
analyze_files,
Skip,
StackItem,
analysis_error,
)
from cwriter import CWriter
from typing import Callable, Mapping, TextIO, Iterator
from lexer import Token
from stack import StackOffset, Stack
ROOT = Path(__file__).parent.parent.parent
DEFAULT_INPUT = (ROOT / "Python/bytecodes.c").absolute()
DEFAULT_INPUT = (ROOT / "Python/bytecodes.c").absolute().as_posix()
def root_relative_path(filename: str) -> str:
return Path(filename).relative_to(ROOT).as_posix()
return Path(filename).absolute().relative_to(ROOT).as_posix()
def write_header(generator: str, source: str, outfile: TextIO) -> None:
def write_header(generator: str, sources: list[str], outfile: TextIO) -> None:
outfile.write(
f"""// This file is generated by {root_relative_path(generator)}
// from:
// {source}
// {", ".join(root_relative_path(src) for src in sources)}
// Do not edit!
"""
)
def emit_to(out: CWriter, tkn_iter: Iterator[Token], end: str) -> None:
parens = 0
for tkn in tkn_iter:
if tkn.kind == end and parens == 0:
return
if tkn.kind == "LPAREN":
parens += 1
if tkn.kind == "RPAREN":
parens -= 1
out.emit(tkn)
def replace_deopt(
out: CWriter,
tkn: Token,
tkn_iter: Iterator[Token],
uop: Uop,
unused: Stack,
inst: Instruction | None,
) -> None:
out.emit_at("DEOPT_IF", tkn)
out.emit(next(tkn_iter))
emit_to(out, tkn_iter, "RPAREN")
next(tkn_iter) # Semi colon
out.emit(", ")
assert inst is not None
assert inst.family is not None
out.emit(inst.family.name)
out.emit(");\n")
def replace_error(
out: CWriter,
tkn: Token,
tkn_iter: Iterator[Token],
uop: Uop,
stack: Stack,
inst: Instruction | None,
) -> None:
out.emit_at("if ", tkn)
out.emit(next(tkn_iter))
emit_to(out, tkn_iter, "COMMA")
label = next(tkn_iter).text
next(tkn_iter) # RPAREN
next(tkn_iter) # Semi colon
out.emit(") ")
c_offset = stack.peek_offset.to_c()
try:
offset = -int(c_offset)
close = ";\n"
except ValueError:
offset = None
out.emit(f"{{ stack_pointer += {c_offset}; ")
close = "; }\n"
out.emit("goto ")
if offset:
out.emit(f"pop_{offset}_")
out.emit(label)
out.emit(close)
def replace_decrefs(
out: CWriter,
tkn: Token,
tkn_iter: Iterator[Token],
uop: Uop,
stack: Stack,
inst: Instruction | None,
) -> None:
next(tkn_iter)
next(tkn_iter)
next(tkn_iter)
out.emit_at("", tkn)
for var in uop.stack.inputs:
if var.name == "unused" or var.name == "null" or var.peek:
continue
if var.size != "1":
out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n")
out.emit(f"Py_DECREF({var.name}[_i]);\n")
out.emit("}\n")
elif var.condition:
out.emit(f"Py_XDECREF({var.name});\n")
else:
out.emit(f"Py_DECREF({var.name});\n")
def replace_store_sp(
out: CWriter,
tkn: Token,
tkn_iter: Iterator[Token],
uop: Uop,
stack: Stack,
inst: Instruction | None,
) -> None:
next(tkn_iter)
next(tkn_iter)
next(tkn_iter)
out.emit_at("", tkn)
stack.flush(out)
out.emit("_PyFrame_SetStackPointer(frame, stack_pointer);\n")
def replace_check_eval_breaker(
out: CWriter,
tkn: Token,
tkn_iter: Iterator[Token],
uop: Uop,
stack: Stack,
inst: Instruction | None,
) -> None:
next(tkn_iter)
next(tkn_iter)
next(tkn_iter)
if not uop.properties.ends_with_eval_breaker:
out.emit_at("CHECK_EVAL_BREAKER();", tkn)
REPLACEMENT_FUNCTIONS = {
"DEOPT_IF": replace_deopt,
"ERROR_IF": replace_error,
"DECREF_INPUTS": replace_decrefs,
"CHECK_EVAL_BREAKER": replace_check_eval_breaker,
"STORE_SP": replace_store_sp,
}
ReplacementFunctionType = Callable[
[CWriter, Token, Iterator[Token], Uop, Stack, Instruction | None], None
]
def emit_tokens(
out: CWriter,
uop: Uop,
stack: Stack,
inst: Instruction | None,
replacement_functions: Mapping[
str, ReplacementFunctionType
] = REPLACEMENT_FUNCTIONS,
) -> None:
tkns = uop.body[1:-1]
if not tkns:
return
tkn_iter = iter(tkns)
out.start_line()
for tkn in tkn_iter:
if tkn.kind == "IDENTIFIER" and tkn.text in replacement_functions:
replacement_functions[tkn.text](out, tkn, tkn_iter, uop, stack, inst)
else:
out.emit(tkn)

2
Tools/cases_generator/opcode_id_generator.py
View File

@ -24,7 +24,7 @@ from typing import TextIO
DEFAULT_OUTPUT = ROOT / "Include/opcode_ids.h"
def generate_opcode_header(filenames: str, analysis: Analysis, outfile: TextIO) -> None:
def generate_opcode_header(filenames: list[str], analysis: Analysis, outfile: TextIO) -> None:
write_header(__file__, filenames, outfile)
out = CWriter(outfile, 0, False)
out.emit("\n")

87
Tools/cases_generator/stack.py
View File

@ -2,6 +2,7 @@ import sys
from analyzer import StackItem
from dataclasses import dataclass
from formatting import maybe_parenthesize
from cwriter import CWriter
def var_size(var: StackItem) -> str:
@ -79,3 +80,89 @@ class StackOffset:
def clear(self) -> None:
self.popped = []
self.pushed = []
class SizeMismatch(Exception):
pass
class Stack:
def __init__(self) -> None:
self.top_offset = StackOffset()
self.base_offset = StackOffset()
self.peek_offset = StackOffset()
self.variables: list[StackItem] = []
self.defined: set[str] = set()
def pop(self, var: StackItem) -> str:
self.top_offset.pop(var)
if not var.peek:
self.peek_offset.pop(var)
indirect = "&" if var.is_array() else ""
if self.variables:
popped = self.variables.pop()
if popped.size != var.size:
raise SizeMismatch(
f"Size mismatch when popping '{popped.name}' from stack to assign to {var.name}. "
f"Expected {var.size} got {popped.size}"
)
if popped.name == var.name:
return ""
elif popped.name == "unused":
self.defined.add(var.name)
return (
f"{var.name} = {indirect}stack_pointer[{self.top_offset.to_c()}];\n"
)
elif var.name == "unused":
return ""
else:
self.defined.add(var.name)
return f"{var.name} = {popped.name};\n"
self.base_offset.pop(var)
if var.name == "unused":
return ""
else:
self.defined.add(var.name)
cast = f"({var.type})" if (not indirect and var.type) else ""
assign = (
f"{var.name} = {cast}{indirect}stack_pointer[{self.base_offset.to_c()}];"
)
if var.condition:
return f"if ({var.condition}) {{ {assign} }}\n"
return f"{assign}\n"
def push(self, var: StackItem) -> str:
self.variables.append(var)
if var.is_array() and var.name not in self.defined and var.name != "unused":
c_offset = self.top_offset.to_c()
self.top_offset.push(var)
self.defined.add(var.name)
return f"{var.name} = &stack_pointer[{c_offset}];\n"
else:
self.top_offset.push(var)
return ""
def flush(self, out: CWriter) -> None:
for var in self.variables:
if not var.peek:
cast = "(PyObject *)" if var.type else ""
if var.name != "unused" and not var.is_array():
if var.condition:
out.emit(f" if ({var.condition}) ")
out.emit(
f"stack_pointer[{self.base_offset.to_c()}] = {cast}{var.name};\n"
)
self.base_offset.push(var)
if self.base_offset.to_c() != self.top_offset.to_c():
print("base", self.base_offset.to_c(), "top", self.top_offset.to_c())
assert False
number = self.base_offset.to_c()
if number != "0":
out.emit(f"stack_pointer += {number};\n")
self.variables = []
self.base_offset.clear()
self.top_offset.clear()
self.peek_offset.clear()
def as_comment(self) -> str:
return f"/* Variables: {[v.name for v in self.variables]}. Base offset: {self.base_offset.to_c()}. Top offset: {self.top_offset.to_c()} */"

228
Tools/cases_generator/tier1_generator.py
View File

@ -21,11 +21,12 @@ from generators_common import (
DEFAULT_INPUT,
ROOT,
write_header,
emit_tokens,
)
from cwriter import CWriter
from typing import TextIO, Iterator
from lexer import Token
from stack import StackOffset
from stack import StackOffset, Stack, SizeMismatch
DEFAULT_OUTPUT = ROOT / "Python/generated_cases.c.h"
@ -34,88 +35,6 @@ DEFAULT_OUTPUT = ROOT / "Python/generated_cases.c.h"
FOOTER = "#undef TIER_ONE\n"
class SizeMismatch(Exception):
pass
class Stack:
def __init__(self) -> None:
self.top_offset = StackOffset()
self.base_offset = StackOffset()
self.peek_offset = StackOffset()
self.variables: list[StackItem] = []
self.defined: set[str] = set()
def pop(self, var: StackItem) -> str:
self.top_offset.pop(var)
if not var.peek:
self.peek_offset.pop(var)
indirect = "&" if var.is_array() else ""
if self.variables:
popped = self.variables.pop()
if popped.size != var.size:
raise SizeMismatch(
f"Size mismatch when popping '{popped.name}' from stack to assign to {var.name}. "
f"Expected {var.size} got {popped.size}"
)
if popped.name == var.name:
return ""
elif popped.name == "unused":
self.defined.add(var.name)
return (
f"{var.name} = {indirect}stack_pointer[{self.top_offset.to_c()}];\n"
)
elif var.name == "unused":
return ""
else:
self.defined.add(var.name)
return f"{var.name} = {popped.name};\n"
self.base_offset.pop(var)
if var.name == "unused":
return ""
else:
self.defined.add(var.name)
assign = f"{var.name} = {indirect}stack_pointer[{self.base_offset.to_c()}];"
if var.condition:
return f"if ({var.condition}) {{ {assign} }}\n"
return f"{assign}\n"
def push(self, var: StackItem) -> str:
self.variables.append(var)
if var.is_array() and var.name not in self.defined and var.name != "unused":
c_offset = self.top_offset.to_c()
self.top_offset.push(var)
self.defined.add(var.name)
return f"{var.name} = &stack_pointer[{c_offset}];\n"
else:
self.top_offset.push(var)
return ""
def flush(self, out: CWriter) -> None:
for var in self.variables:
if not var.peek:
if var.name != "unused" and not var.is_array():
if var.condition:
out.emit(f" if ({var.condition}) ")
out.emit(
f"stack_pointer[{self.base_offset.to_c()}] = {var.name};\n"
)
self.base_offset.push(var)
if self.base_offset.to_c() != self.top_offset.to_c():
print("base", self.base_offset.to_c(), "top", self.top_offset.to_c())
assert False
number = self.base_offset.to_c()
if number != "0":
out.emit(f"stack_pointer += {number};\n")
self.variables = []
self.base_offset.clear()
self.top_offset.clear()
self.peek_offset.clear()
def as_comment(self) -> str:
return f"/* Variables: {[v.name for v in self.variables]}. Base offset: {self.base_offset.to_c()}. Top offset: {self.top_offset.to_c()} */"
def declare_variables(inst: Instruction, out: CWriter) -> None:
variables = {"unused"}
for uop in inst.parts:
@ -138,145 +57,6 @@ def declare_variables(inst: Instruction, out: CWriter) -> None:
out.emit(f"{type}{var.name};\n")
def emit_to(out: CWriter, tkn_iter: Iterator[Token], end: str) -> None:
parens = 0
for tkn in tkn_iter:
if tkn.kind == end and parens == 0:
return
if tkn.kind == "LPAREN":
parens += 1
if tkn.kind == "RPAREN":
parens -= 1
out.emit(tkn)
def replace_deopt(
out: CWriter,
tkn: Token,
tkn_iter: Iterator[Token],
uop: Uop,
unused: Stack,
inst: Instruction,
) -> None:
out.emit_at("DEOPT_IF", tkn)
out.emit(next(tkn_iter))
emit_to(out, tkn_iter, "RPAREN")
next(tkn_iter) # Semi colon
out.emit(", ")
assert inst.family is not None
out.emit(inst.family.name)
out.emit(");\n")
def replace_error(
out: CWriter,
tkn: Token,
tkn_iter: Iterator[Token],
uop: Uop,
stack: Stack,
inst: Instruction,
) -> None:
out.emit_at("if ", tkn)
out.emit(next(tkn_iter))
emit_to(out, tkn_iter, "COMMA")
label = next(tkn_iter).text
next(tkn_iter) # RPAREN
next(tkn_iter) # Semi colon
out.emit(") ")
c_offset = stack.peek_offset.to_c()
try:
offset = -int(c_offset)
close = ";\n"
except ValueError:
offset = None
out.emit(f"{{ stack_pointer += {c_offset}; ")
close = "; }\n"
out.emit("goto ")
if offset:
out.emit(f"pop_{offset}_")
out.emit(label)
out.emit(close)
def replace_decrefs(
out: CWriter,
tkn: Token,
tkn_iter: Iterator[Token],
uop: Uop,
stack: Stack,
inst: Instruction,
) -> None:
next(tkn_iter)
next(tkn_iter)
next(tkn_iter)
out.emit_at("", tkn)
for var in uop.stack.inputs:
if var.name == "unused" or var.name == "null" or var.peek:
continue
if var.size != "1":
out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n")
out.emit(f"Py_DECREF({var.name}[_i]);\n")
out.emit("}\n")
elif var.condition:
out.emit(f"Py_XDECREF({var.name});\n")
else:
out.emit(f"Py_DECREF({var.name});\n")
def replace_store_sp(
out: CWriter,
tkn: Token,
tkn_iter: Iterator[Token],
uop: Uop,
stack: Stack,
inst: Instruction,
) -> None:
next(tkn_iter)
next(tkn_iter)
next(tkn_iter)
out.emit_at("", tkn)
stack.flush(out)
out.emit("_PyFrame_SetStackPointer(frame, stack_pointer);\n")
def replace_check_eval_breaker(
out: CWriter,
tkn: Token,
tkn_iter: Iterator[Token],
uop: Uop,
stack: Stack,
inst: Instruction,
) -> None:
next(tkn_iter)
next(tkn_iter)
next(tkn_iter)
if not uop.properties.ends_with_eval_breaker:
out.emit_at("CHECK_EVAL_BREAKER();", tkn)
REPLACEMENT_FUNCTIONS = {
"DEOPT_IF": replace_deopt,
"ERROR_IF": replace_error,
"DECREF_INPUTS": replace_decrefs,
"CHECK_EVAL_BREAKER": replace_check_eval_breaker,
"STORE_SP": replace_store_sp,
}
# Move this to formatter
def emit_tokens(out: CWriter, uop: Uop, stack: Stack, inst: Instruction) -> None:
tkns = uop.body[1:-1]
if not tkns:
return
tkn_iter = iter(tkns)
out.start_line()
for tkn in tkn_iter:
if tkn.kind == "IDENTIFIER" and tkn.text in REPLACEMENT_FUNCTIONS:
REPLACEMENT_FUNCTIONS[tkn.text](out, tkn, tkn_iter, uop, stack, inst)
else:
out.emit(tkn)
def write_uop(
uop: Part, out: CWriter, offset: int, stack: Stack, inst: Instruction, braces: bool
) -> int:
@ -334,7 +114,7 @@ def uses_this(inst: Instruction) -> bool:
def generate_tier1(
filenames: str, analysis: Analysis, outfile: TextIO, lines: bool
filenames: list[str], analysis: Analysis, outfile: TextIO, lines: bool
) -> None:
write_header(__file__, filenames, outfile)
outfile.write(
@ -404,7 +184,7 @@ arg_parser.add_argument(
if __name__ == "__main__":
args = arg_parser.parse_args()
if len(args.input) == 0:
args.input.append(DEFAULT_INPUT.as_posix())
args.input.append(DEFAULT_INPUT)
data = analyze_files(args.input)
with open(args.output, "w") as outfile:
generate_tier1(args.input, data, outfile, args.emit_line_directives)

202
Tools/cases_generator/tier2_generator.py Normal file
View File

@ -0,0 +1,202 @@
"""Generate the cases for the tier 2 interpreter.
Reads the instruction definitions from bytecodes.c.
Writes the cases to executor_cases.c.h, which is #included in ceval.c.
"""
import argparse
import os.path
import sys
from analyzer import (
Analysis,
Instruction,
Uop,
Part,
analyze_files,
Skip,
StackItem,
analysis_error,
)
from generators_common import (
DEFAULT_INPUT,
ROOT,
write_header,
emit_tokens,
emit_to,
REPLACEMENT_FUNCTIONS,
)
from cwriter import CWriter
from typing import TextIO, Iterator
from lexer import Token
from stack import StackOffset, Stack, SizeMismatch
DEFAULT_OUTPUT = ROOT / "Python/executor_cases.c.h"
def declare_variables(uop: Uop, out: CWriter) -> None:
variables = {"unused"}
for var in reversed(uop.stack.inputs):
if var.name not in variables:
type = var.type if var.type else "PyObject *"
variables.add(var.name)
if var.condition:
out.emit(f"{type}{var.name} = NULL;\n")
else:
out.emit(f"{type}{var.name};\n")
for var in uop.stack.outputs:
if var.name not in variables:
variables.add(var.name)
type = var.type if var.type else "PyObject *"
if var.condition:
out.emit(f"{type}{var.name} = NULL;\n")
else:
out.emit(f"{type}{var.name};\n")
def tier2_replace_error(
out: CWriter,
tkn: Token,
tkn_iter: Iterator[Token],
uop: Uop,
stack: Stack,
inst: Instruction | None,
) -> None:
out.emit_at("if ", tkn)
out.emit(next(tkn_iter))
emit_to(out, tkn_iter, "COMMA")
label = next(tkn_iter).text
next(tkn_iter) # RPAREN
next(tkn_iter) # Semi colon
out.emit(") ")
c_offset = stack.peek_offset.to_c()
try:
offset = -int(c_offset)
close = ";\n"
except ValueError:
offset = None
out.emit(f"{{ stack_pointer += {c_offset}; ")
close = "; }\n"
out.emit("goto ")
if offset:
out.emit(f"pop_{offset}_")
out.emit(label + "_tier_two")
out.emit(close)
def tier2_replace_deopt(
out: CWriter,
tkn: Token,
tkn_iter: Iterator[Token],
uop: Uop,
unused: Stack,
inst: Instruction | None,
) -> None:
out.emit_at("if ", tkn)
out.emit(next(tkn_iter))
emit_to(out, tkn_iter, "RPAREN")
next(tkn_iter) # Semi colon
out.emit(") goto deoptimize;\n")
TIER2_REPLACEMENT_FUNCTIONS = REPLACEMENT_FUNCTIONS.copy()
TIER2_REPLACEMENT_FUNCTIONS["ERROR_IF"] = tier2_replace_error
TIER2_REPLACEMENT_FUNCTIONS["DEOPT_IF"] = tier2_replace_deopt
def is_super(uop: Uop) -> bool:
for tkn in uop.body:
if tkn.kind == "IDENTIFIER" and tkn.text == "oparg1":
return True
return False
def write_uop(uop: Uop, out: CWriter, stack: Stack) -> None:
try:
out.start_line()
if uop.properties.oparg:
out.emit("oparg = CURRENT_OPARG();\n")
for var in reversed(uop.stack.inputs):
out.emit(stack.pop(var))
if not uop.properties.stores_sp:
for i, var in enumerate(uop.stack.outputs):
out.emit(stack.push(var))
for cache in uop.caches:
if cache.name != "unused":
if cache.size == 4:
type = "PyObject *"
else:
type = f"uint{cache.size*16}_t"
out.emit(f"{type} {cache.name} = ({type})CURRENT_OPERAND();\n")
emit_tokens(out, uop, stack, None, TIER2_REPLACEMENT_FUNCTIONS)
if uop.properties.stores_sp:
for i, var in enumerate(uop.stack.outputs):
out.emit(stack.push(var))
except SizeMismatch as ex:
raise analysis_error(ex.args[0], uop.body[0])
SKIPS = ("_EXTENDED_ARG",)
def generate_tier2(
filenames: list[str], analysis: Analysis, outfile: TextIO, lines: bool
) -> None:
write_header(__file__, filenames, outfile)
outfile.write(
"""
#ifdef TIER_ONE
#error "This file is for Tier 2 only"
#endif
#define TIER_TWO 2
"""
)
out = CWriter(outfile, 2, lines)
out.emit("\n")
for name, uop in analysis.uops.items():
if uop.properties.tier_one_only:
continue
if is_super(uop):
continue
if not uop.is_viable():
out.emit(f"/* {uop.name} is not a viable micro-op for tier 2 */\n\n")
continue
out.emit(f"case {uop.name}: {{\n")
declare_variables(uop, out)
stack = Stack()
write_uop(uop, out, stack)
out.start_line()
if not uop.properties.always_exits:
stack.flush(out)
if uop.properties.ends_with_eval_breaker:
out.emit("CHECK_EVAL_BREAKER();\n")
out.emit("break;\n")
out.start_line()
out.emit("}")
out.emit("\n\n")
outfile.write("#undef TIER_TWO\n")
arg_parser = argparse.ArgumentParser(
description="Generate the code for the tier 2 interpreter.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
arg_parser.add_argument(
"-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT
)
arg_parser.add_argument(
"-l", "--emit-line-directives", help="Emit #line directives", action="store_true"
)
arg_parser.add_argument(
"input", nargs=argparse.REMAINDER, help="Instruction definition file(s)"
)
if __name__ == "__main__":
args = arg_parser.parse_args()
if len(args.input) == 0:
args.input.append(DEFAULT_INPUT)
data = analyze_files(args.input)
with open(args.output, "w") as outfile:
generate_tier2(args.input, data, outfile, args.emit_line_directives)

13
Tools/cases_generator/uop_id_generator.py
View File

@ -24,8 +24,11 @@ from typing import TextIO
DEFAULT_OUTPUT = ROOT / "Include/internal/pycore_uop_ids.h"
OMIT = {"_CACHE", "_RESERVED", "_EXTENDED_ARG"}
def generate_uop_ids(
filenames: str, analysis: Analysis, outfile: TextIO, distinct_namespace: bool
filenames: list[str], analysis: Analysis, outfile: TextIO, distinct_namespace: bool
) -> None:
write_header(__file__, filenames, outfile)
out = CWriter(outfile, 0, False)
@ -45,11 +48,15 @@ extern "C" {
next_id += 1
out.emit(f"#define _SET_IP {next_id}\n")
next_id += 1
PRE_DEFINED = {"_EXIT_TRACE", "_SET_IP", "_CACHE", "_RESERVED", "_EXTENDED_ARG"}
PRE_DEFINED = {"_EXIT_TRACE", "_SET_IP"}
for uop in analysis.uops.values():
if uop.name in PRE_DEFINED:
continue
# TODO: We should omit all tier-1 only uops, but
# generate_cases.py still generates code for those.
if uop.name in OMIT:
continue
if uop.implicitly_created and not distinct_namespace:
out.emit(f"#define {uop.name} {uop.name[1:]}\n")
else:
@ -85,7 +92,7 @@ arg_parser.add_argument(
if __name__ == "__main__":
args = arg_parser.parse_args()
if len(args.input) == 0:
args.input.append(DEFAULT_INPUT.as_posix())
args.input.append(DEFAULT_INPUT)
data = analyze_files(args.input)
with open(args.output, "w") as outfile:
generate_uop_ids(args.input, data, outfile, args.namespace)