Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

GH-98831: Implement super-instruction generation (#99084) 

Co-authored-by: C.A.M. Gerlach <CAM.Gerlach@Gerlach.CAM>
This commit is contained in:
Guido van Rossum 2022-11-06 09:40:47 -08:00 committed by GitHub
parent ede6cb2615
commit 7dcd28eb41
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 163 additions and 138 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
Misc/NEWS.d/next/Build/2022-10-28-22-24-26.gh-issue-98831.IXRCRX.rst
View File

@ -1 +1,9 @@
We have new tooling, in ``Tools/cases_generator``, to generate the interpreter switch from a list of opcode definitions.
Add new tooling, in ``Tools/cases_generator``,
to generate the interpreter switch statement from a list of opcode definitions.
This only affects adding, modifying or removing instruction definitions.
The instruction definitions now live in ``Python/bytecodes.c``,
in the form of a `custom DSL (under development)
<https://github.com/faster-cpython/ideas/blob/main/3.12/interpreter_definition.md>`__.
The tooling reads this file and writes ``Python/generated_cases.c.h``,
which is then included by ``Python/ceval.c`` to provide most of the cases
of the main interpreter switch.

67
Python/bytecodes.c
View File

@ -69,6 +69,7 @@ do { \
#define DISPATCH() ((void)0)
#define inst(name) case name:
#define super(name) static int SUPER_##name
#define family(name) static int family_##name
#define NAME_ERROR_MSG \
@ -158,67 +159,11 @@ dummy_func(
SETLOCAL(oparg, value);
}
// stack effect: ( -- __0, __1)
inst(LOAD_FAST__LOAD_FAST) {
PyObject *value = GETLOCAL(oparg);
assert(value != NULL);
NEXTOPARG();
next_instr++;
Py_INCREF(value);
PUSH(value);
value = GETLOCAL(oparg);
assert(value != NULL);
Py_INCREF(value);
PUSH(value);
}
// stack effect: ( -- __0, __1)
inst(LOAD_FAST__LOAD_CONST) {
PyObject *value = GETLOCAL(oparg);
assert(value != NULL);
NEXTOPARG();
next_instr++;
Py_INCREF(value);
PUSH(value);
value = GETITEM(consts, oparg);
Py_INCREF(value);
PUSH(value);
}
// stack effect: ( -- )
inst(STORE_FAST__LOAD_FAST) {
PyObject *value = POP();
SETLOCAL(oparg, value);
NEXTOPARG();
next_instr++;
value = GETLOCAL(oparg);
assert(value != NULL);
Py_INCREF(value);
PUSH(value);
}
// stack effect: (__0, __1 -- )
inst(STORE_FAST__STORE_FAST) {
PyObject *value = POP();
SETLOCAL(oparg, value);
NEXTOPARG();
next_instr++;
value = POP();
SETLOCAL(oparg, value);
}
// stack effect: ( -- __0, __1)
inst(LOAD_CONST__LOAD_FAST) {
PyObject *value = GETITEM(consts, oparg);
NEXTOPARG();
next_instr++;
Py_INCREF(value);
PUSH(value);
value = GETLOCAL(oparg);
assert(value != NULL);
Py_INCREF(value);
PUSH(value);
}
super(LOAD_FAST__LOAD_FAST) = LOAD_FAST + LOAD_FAST;
super(LOAD_FAST__LOAD_CONST) = LOAD_FAST + LOAD_CONST;
super(STORE_FAST__LOAD_FAST) = STORE_FAST + LOAD_FAST;
super(STORE_FAST__STORE_FAST) = STORE_FAST + STORE_FAST;
super (LOAD_CONST__LOAD_FAST) = LOAD_CONST + LOAD_FAST;
// stack effect: (__0 -- )
inst(POP_TOP) {

144
Python/generated_cases.c.h generated
View File

@ -57,68 +57,6 @@
DISPATCH();
}
TARGET(LOAD_FAST__LOAD_FAST) {
PyObject *value = GETLOCAL(oparg);
assert(value != NULL);
NEXTOPARG();
next_instr++;
Py_INCREF(value);
PUSH(value);
value = GETLOCAL(oparg);
assert(value != NULL);
Py_INCREF(value);
PUSH(value);
DISPATCH();
}
TARGET(LOAD_FAST__LOAD_CONST) {
PyObject *value = GETLOCAL(oparg);
assert(value != NULL);
NEXTOPARG();
next_instr++;
Py_INCREF(value);
PUSH(value);
value = GETITEM(consts, oparg);
Py_INCREF(value);
PUSH(value);
DISPATCH();
}
TARGET(STORE_FAST__LOAD_FAST) {
PyObject *value = POP();
SETLOCAL(oparg, value);
NEXTOPARG();
next_instr++;
value = GETLOCAL(oparg);
assert(value != NULL);
Py_INCREF(value);
PUSH(value);
DISPATCH();
}
TARGET(STORE_FAST__STORE_FAST) {
PyObject *value = POP();
SETLOCAL(oparg, value);
NEXTOPARG();
next_instr++;
value = POP();
SETLOCAL(oparg, value);
DISPATCH();
}
TARGET(LOAD_CONST__LOAD_FAST) {
PyObject *value = GETITEM(consts, oparg);
NEXTOPARG();
next_instr++;
Py_INCREF(value);
PUSH(value);
value = GETLOCAL(oparg);
assert(value != NULL);
Py_INCREF(value);
PUSH(value);
DISPATCH();
}
TARGET(POP_TOP) {
PyObject *value = POP();
Py_DECREF(value);
@ -3900,3 +3838,85 @@
TARGET(CACHE) {
Py_UNREACHABLE();
}
TARGET(LOAD_FAST__LOAD_FAST) {
{
PyObject *value = GETLOCAL(oparg);
assert(value != NULL);
Py_INCREF(value);
PUSH(value);
}
NEXTOPARG();
next_instr++;
{
PyObject *value = GETLOCAL(oparg);
assert(value != NULL);
Py_INCREF(value);
PUSH(value);
}
DISPATCH();
}
TARGET(LOAD_FAST__LOAD_CONST) {
{
PyObject *value = GETLOCAL(oparg);
assert(value != NULL);
Py_INCREF(value);
PUSH(value);
}
NEXTOPARG();
next_instr++;
{
PyObject *value = GETITEM(consts, oparg);
Py_INCREF(value);
PUSH(value);
}
DISPATCH();
}
TARGET(STORE_FAST__LOAD_FAST) {
{
PyObject *value = POP();
SETLOCAL(oparg, value);
}
NEXTOPARG();
next_instr++;
{
PyObject *value = GETLOCAL(oparg);
assert(value != NULL);
Py_INCREF(value);
PUSH(value);
}
DISPATCH();
}
TARGET(STORE_FAST__STORE_FAST) {
{
PyObject *value = POP();
SETLOCAL(oparg, value);
}
NEXTOPARG();
next_instr++;
{
PyObject *value = POP();
SETLOCAL(oparg, value);
}
DISPATCH();
}
TARGET(LOAD_CONST__LOAD_FAST) {
{
PyObject *value = GETITEM(consts, oparg);
Py_INCREF(value);
PUSH(value);
}
NEXTOPARG();
next_instr++;
{
PyObject *value = GETLOCAL(oparg);
assert(value != NULL);
Py_INCREF(value);
PUSH(value);
}
DISPATCH();
}

44
Tools/cases_generator/generate_cases.py
View File

@ -11,7 +11,7 @@ import re
import sys
import parser
from parser import InstDef
from parser import InstDef # TODO: Use parser.InstDef
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument("-i", "--input", type=str, default="Python/bytecodes.c")
@ -29,19 +29,24 @@ def eopen(filename: str, mode: str = "r"):
return open(filename, mode)
def parse_cases(src: str, filename: str|None = None) -> tuple[list[InstDef], list[parser.Family]]:
def parse_cases(
src: str, filename: str|None = None
) -> tuple[list[InstDef], list[parser.Super], list[parser.Family]]:
psr = parser.Parser(src, filename=filename)
instrs: list[InstDef] = []
supers: list[parser.Super] = []
families: list[parser.Family] = []
while not psr.eof():
if inst := psr.inst_def():
assert inst.block
instrs.append(InstDef(inst.name, inst.inputs, inst.outputs, inst.block))
instrs.append(inst)
elif sup := psr.super_def():
supers.append(sup)
elif fam := psr.family_def():
families.append(fam)
else:
raise psr.make_syntax_error(f"Unexpected token")
return instrs, families
return instrs, supers, families
def always_exits(block: parser.Block) -> bool:
@ -62,7 +67,7 @@ def always_exits(block: parser.Block) -> bool:
return line.startswith(("goto ", "return ", "DISPATCH", "GO_TO_", "Py_UNREACHABLE()"))
def write_cases(f: io.TextIOBase, instrs: list[InstDef]):
def write_cases(f: io.TextIOBase, instrs: list[InstDef], supers: list[parser.Super]):
predictions = set()
for inst in instrs:
for target in re.findall(r"(?:PREDICT|GO_TO_INSTRUCTION)\((\w+)\)", inst.block.text):
@ -70,8 +75,10 @@ def write_cases(f: io.TextIOBase, instrs: list[InstDef]):
indent = " "
f.write(f"// This file is generated by {os.path.relpath(__file__)}\n")
f.write("// Do not edit!\n")
instr_index: dict[str, InstDef] = {}
for instr in instrs:
assert isinstance(instr, InstDef)
instr_index[instr.name] = instr
f.write(f"\n{indent}TARGET({instr.name}) {{\n")
if instr.name in predictions:
f.write(f"{indent} PREDICTED({instr.name});\n")
@ -102,6 +109,22 @@ def write_cases(f: io.TextIOBase, instrs: list[InstDef]):
# Write trailing '}'
f.write(f"{indent}}}\n")
for sup in supers:
assert isinstance(sup, parser.Super)
components = [instr_index[name] for name in sup.ops]
f.write(f"\n{indent}TARGET({sup.name}) {{\n")
for i, instr in enumerate(components):
if i > 0:
f.write(f"{indent} NEXTOPARG();\n")
f.write(f"{indent} next_instr++;\n")
text = instr.block.to_text(-4)
textlines = text.splitlines(True)
textlines = [line for line in textlines if not line.strip().startswith("PREDICTED(")]
text = "".join(textlines)
f.write(f"{indent} {text.strip()}\n")
f.write(f"{indent} DISPATCH();\n")
f.write(f"{indent}}}\n")
def main():
args = arg_parser.parse_args()
@ -110,21 +133,22 @@ def main():
begin = srclines.index("// BEGIN BYTECODES //")
end = srclines.index("// END BYTECODES //")
src = "\n".join(srclines[begin+1 : end])
instrs, families = parse_cases(src, filename=args.input)
ninstrs = nfamilies = 0
instrs, supers, families = parse_cases(src, filename=args.input)
ninstrs = nsupers = nfamilies = 0
if not args.quiet:
ninstrs = len(instrs)
nsupers = len(supers)
nfamilies = len(families)
print(
f"Read {ninstrs} instructions "
f"Read {ninstrs} instructions, {nsupers} supers, "
f"and {nfamilies} families from {args.input}",
file=sys.stderr,
)
with eopen(args.output, "w") as f:
write_cases(f, instrs)
write_cases(f, instrs, supers)
if not args.quiet:
print(
f"Wrote {ninstrs} instructions to {args.output}",
f"Wrote {ninstrs + nsupers} instructions to {args.output}",
file=sys.stderr,
)

36
Tools/cases_generator/parser.py
View File

@ -39,13 +39,16 @@ class Node:
@property
def text(self) -> str:
return self.to_text()
def to_text(self, dedent: int = 0) -> str:
context = self.context
if not context:
return ""
tokens = context.owner.tokens
begin = context.begin
end = context.end
return lx.to_text(tokens[begin:end])
return lx.to_text(tokens[begin:end], dedent)
@dataclass
@ -61,6 +64,12 @@ class InstDef(Node):
block: Block | None
@dataclass
class Super(Node):
name: str
ops: list[str]
@dataclass
class Family(Node):
name: str
@ -156,17 +165,36 @@ class Parser(PLexer):
return self.input() # TODO: They're not quite the same.
@contextual
def family_def(self) -> Family | None:
def super_def(self) -> Super | None:
if (tkn := self.expect(lx.IDENTIFIER)) and tkn.text == "super":
if self.expect(lx.LPAREN):
if (tkn := self.expect(lx.IDENTIFIER)):
if self.expect(lx.RPAREN):
if self.expect(lx.EQUALS):
if ops := self.ops():
res = Super(tkn.text, ops)
return res
def ops(self) -> list[str] | None:
here = self.getpos()
if tkn := self.expect(lx.IDENTIFIER):
ops = [tkn.text]
while self.expect(lx.PLUS):
if tkn := self.require(lx.IDENTIFIER):
ops.append(tkn.text)
self.require(lx.SEMI)
return ops
@contextual
def family_def(self) -> Family | None:
if (tkn := self.expect(lx.IDENTIFIER)) and tkn.text == "family":
if self.expect(lx.LPAREN):
if (tkn := self.expect(lx.IDENTIFIER)):
name = tkn.text
if self.expect(lx.RPAREN):
if self.expect(lx.EQUALS):
if members := self.members():
if self.expect(lx.SEMI):
return Family(name, members)
return Family(tkn.text, members)
return None
def members(self):