Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

GH-118306: Update JIT to use LLVM 18 (GH-118307)

This commit is contained in:
Savannah Ostrowski 2024-04-29 14:09:16 -07:00 committed by GitHub
parent 8e4fb5d260
commit 8b56d82c59
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 55 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
.github/workflows/jit.yml vendored
View File

@ -41,7 +41,7 @@ jobs:
- true
- false
llvm:
- 16
- 18
include:
- target: i686-pc-windows-msvc/msvc
architecture: Win32
@ -94,6 +94,7 @@ jobs:
- name: Native Windows
if: runner.os == 'Windows' && matrix.architecture != 'ARM64'
run: |
choco upgrade llvm -y
choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}
./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '--pgo' }} -p ${{ matrix.architecture }}
./PCbuild/rt.bat ${{ matrix.debug && '-d' }} -p ${{ matrix.architecture }} -q --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3
@ -102,27 +103,31 @@ jobs:
- name: Emulated Windows
if: runner.os == 'Windows' && matrix.architecture == 'ARM64'
run: |
choco upgrade llvm -y
choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}
./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }}
- name: Native macOS
if: runner.os == 'macOS'
run: |
brew update
brew install llvm@${{ matrix.llvm }}
SDKROOT="$(xcrun --show-sdk-path)" \
./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }}
make all --jobs 4
./python.exe -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3
# --with-lto has been removed temporarily as a result of an open issue in LLVM 18 (see https://github.com/llvm/llvm-project/issues/87553)
- name: Native Linux
if: runner.os == 'Linux' && matrix.architecture == 'x86_64'
run: |
sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }}
export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH"
./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }}
./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations' }}
make all --jobs 4
./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3
# --with-lto has been removed temporarily as a result of an open issue in LLVM 18 (see https://github.com/llvm/llvm-project/issues/87553)
- name: Emulated Linux
if: runner.os == 'Linux' && matrix.architecture != 'x86_64'
run: |
@ -139,6 +144,6 @@ jobs:
CC="${{ matrix.compiler == 'clang' && 'clang --target=$HOST' || '$HOST-gcc' }}" \
CPP="$CC --preprocess" \
HOSTRUNNER=qemu-${{ matrix.architecture }} \
./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes
./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations ' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes
make all --jobs 4
./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3

1
Misc/NEWS.d/next/Core and Builtins/2024-04-26-05-38-18.gh-issue-118306.vRUEOU.rst Normal file
View File

@ -0,0 +1 @@
Update JIT compilation to use LLVM 18

3
Python/jit.c
View File

@ -234,6 +234,7 @@ patch(unsigned char *base, const Stencil *stencil, uintptr_t patches[])
assert((int64_t)value < (1LL << 31));
*loc32 = (uint32_t)value;
continue;
case HoleKind_ARM64_RELOC_BRANCH26:
case HoleKind_IMAGE_REL_ARM64_BRANCH26:
case HoleKind_R_AARCH64_CALL26:
case HoleKind_R_AARCH64_JUMP26:
@ -278,6 +279,7 @@ patch(unsigned char *base, const Stencil *stencil, uintptr_t patches[])
case HoleKind_ARM64_RELOC_GOT_LOAD_PAGE21:
case HoleKind_IMAGE_REL_ARM64_PAGEBASE_REL21:
case HoleKind_R_AARCH64_ADR_GOT_PAGE:
case HoleKind_R_AARCH64_ADR_PREL_PG_HI21:
// 21-bit count of pages between this page and an absolute address's
// page... I know, I know, it's weird. Pairs nicely with
// ARM64_RELOC_GOT_LOAD_PAGEOFF12 (below).
@ -341,6 +343,7 @@ patch(unsigned char *base, const Stencil *stencil, uintptr_t patches[])
case HoleKind_ARM64_RELOC_PAGEOFF12:
case HoleKind_IMAGE_REL_ARM64_PAGEOFFSET_12A:
case HoleKind_IMAGE_REL_ARM64_PAGEOFFSET_12L:
case HoleKind_R_AARCH64_ADD_ABS_LO12_NC:
case HoleKind_R_AARCH64_LD64_GOT_LO12_NC:
// 12-bit low part of an absolute address. Pairs nicely with
// ARM64_RELOC_GOT_LOAD_PAGE21 (above).

10
Tools/jit/README.md
View File

@ -23,17 +23,21 @@ sudo ./llvm.sh 16
### macOS
Install LLVM 16 with [Homebrew](https://brew.sh):
Install LLVM 18 with [Homebrew](https://brew.sh):
```sh
brew install llvm@16
brew install llvm@18
```
Homebrew won't add any of the tools to your `$PATH`. That's okay; the build script knows how to find them.
### Windows
Install LLVM 16 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=16), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".**
Install LLVM 18 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=18), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".**
### Dev Containers
If you are working CPython in a [Codespaces instance](https://devguide.python.org/getting-started/setup-building/#using-codespaces), there's no need to install LLVM as the Fedora 40 base image includes LLVM 18 out of the box.
## Building

2
Tools/jit/_llvm.py
View File

@ -7,7 +7,7 @@ import shlex
import subprocess
import typing
_LLVM_VERSION = 16
_LLVM_VERSION = 18
_LLVM_VERSION_PATTERN = re.compile(rf"version\s+{_LLVM_VERSION}\.\d+\.\d+\s+")
_P = typing.ParamSpec("_P")

9
Tools/jit/_schema.py
View File

@ -2,6 +2,7 @@
import typing
HoleKind: typing.TypeAlias = typing.Literal[
"ARM64_RELOC_BRANCH26",
"ARM64_RELOC_GOT_LOAD_PAGE21",
"ARM64_RELOC_GOT_LOAD_PAGEOFF12",
"ARM64_RELOC_PAGE21",
@ -16,8 +17,10 @@ HoleKind: typing.TypeAlias = typing.Literal[
"IMAGE_REL_I386_REL32",
"R_AARCH64_ABS64",
"R_AARCH64_ADR_GOT_PAGE",
"R_AARCH64_ADR_PREL_PG_HI21",
"R_AARCH64_CALL26",
"R_AARCH64_JUMP26",
"R_AARCH64_ADD_ABS_LO12_NC",
"R_AARCH64_LD64_GOT_LO12_NC",
"R_AARCH64_MOVW_UABS_G0_NC",
"R_AARCH64_MOVW_UABS_G1_NC",
@ -68,12 +71,12 @@ class _COFFSymbol(typing.TypedDict):
class _ELFSymbol(typing.TypedDict):
Name: dict[typing.Literal["Value"], str]
Name: dict[typing.Literal["Name"], str]
Value: int
class _MachOSymbol(typing.TypedDict):
Name: dict[typing.Literal["Value"], str]
Name: dict[typing.Literal["Name"], str]
Value: int
@ -99,7 +102,7 @@ class ELFSection(typing.TypedDict):
Relocations: list[dict[typing.Literal["Relocation"], ELFRelocation]]
SectionData: dict[typing.Literal["Bytes"], list[int]]
Symbols: list[dict[typing.Literal["Symbol"], _ELFSymbol]]
Type: dict[typing.Literal["Value"], str]
Type: dict[typing.Literal["Name"], str]
class MachOSection(typing.TypedDict):

4
Tools/jit/_stencils.py
View File

@ -203,7 +203,9 @@ class StencilGroup:
"""Fix up all GOT and internal relocations for this stencil group."""
for hole in self.code.holes.copy():
if (
hole.kind in {"R_AARCH64_CALL26", "R_AARCH64_JUMP26"}
hole.kind in {
"R_AARCH64_CALL26", "R_AARCH64_JUMP26", "ARM64_RELOC_BRANCH26"
}
and hole.value is HoleValue.ZERO
):
self.code.pad(alignment)

52
Tools/jit/_targets.py
View File

@ -221,7 +221,7 @@ class _COFF(
case {
"Offset": offset,
"Symbol": s,
"Type": {"Value": "IMAGE_REL_I386_DIR32" as kind},
"Type": {"Name": "IMAGE_REL_I386_DIR32" as kind},
}:
offset += base
value, symbol = self._unwrap_dllimport(s)
@ -230,7 +230,7 @@ class _COFF(
"Offset": offset,
"Symbol": s,
"Type": {
"Value": "IMAGE_REL_AMD64_REL32" | "IMAGE_REL_I386_REL32" as kind
"Name": "IMAGE_REL_AMD64_REL32" | "IMAGE_REL_I386_REL32" as kind
},
}:
offset += base
@ -242,7 +242,7 @@ class _COFF(
"Offset": offset,
"Symbol": s,
"Type": {
"Value": "IMAGE_REL_ARM64_BRANCH26"
"Name": "IMAGE_REL_ARM64_BRANCH26"
| "IMAGE_REL_ARM64_PAGEBASE_REL21"
| "IMAGE_REL_ARM64_PAGEOFFSET_12A"
| "IMAGE_REL_ARM64_PAGEOFFSET_12L" as kind
@ -262,7 +262,7 @@ class _ELF(
def _handle_section(
self, section: _schema.ELFSection, group: _stencils.StencilGroup
) -> None:
section_type = section["Type"]["Value"]
section_type = section["Type"]["Name"]
flags = {flag["Name"] for flag in section["Flags"]["Flags"]}
if section_type == "SHT_RELA":
assert "SHF_INFO_LINK" in flags, flags
@ -290,7 +290,7 @@ class _ELF(
for wrapped_symbol in section["Symbols"]:
symbol = wrapped_symbol["Symbol"]
offset = len(stencil.body) + symbol["Value"]
name = symbol["Name"]["Value"]
name = symbol["Name"]["Name"]
name = name.removeprefix(self.prefix)
group.symbols[name] = value, offset
stencil.body.extend(section["SectionData"]["Bytes"])
@ -312,9 +312,9 @@ class _ELF(
case {
"Addend": addend,
"Offset": offset,
"Symbol": {"Value": s},
"Symbol": {"Name": s},
"Type": {
"Value": "R_AARCH64_ADR_GOT_PAGE"
"Name": "R_AARCH64_ADR_GOT_PAGE"
| "R_AARCH64_LD64_GOT_LO12_NC"
| "R_X86_64_GOTPCREL"
| "R_X86_64_GOTPCRELX"
@ -327,8 +327,8 @@ class _ELF(
case {
"Addend": addend,
"Offset": offset,
"Symbol": {"Value": s},
"Type": {"Value": kind},
"Symbol": {"Name": s},
"Type": {"Name": kind},
}:
offset += base
s = s.removeprefix(self.prefix)
@ -371,7 +371,7 @@ class _MachO(
for wrapped_symbol in section["Symbols"]:
symbol = wrapped_symbol["Symbol"]
offset = symbol["Value"] - start_address
name = symbol["Name"]["Value"]
name = symbol["Name"]["Name"]
name = name.removeprefix(self.prefix)
group.symbols[name] = value, offset
assert "Relocations" in section
@ -387,9 +387,9 @@ class _MachO(
match relocation:
case {
"Offset": offset,
"Symbol": {"Value": s},
"Symbol": {"Name": s},
"Type": {
"Value": "ARM64_RELOC_GOT_LOAD_PAGE21"
"Name": "ARM64_RELOC_GOT_LOAD_PAGE21"
| "ARM64_RELOC_GOT_LOAD_PAGEOFF12" as kind
},
}:
@ -399,8 +399,8 @@ class _MachO(
addend = 0
case {
"Offset": offset,
"Symbol": {"Value": s},
"Type": {"Value": "X86_64_RELOC_GOT" | "X86_64_RELOC_GOT_LOAD" as kind},
"Symbol": {"Name": s},
"Type": {"Name": "X86_64_RELOC_GOT" | "X86_64_RELOC_GOT_LOAD" as kind},
}:
offset += base
s = s.removeprefix(self.prefix)
@ -410,13 +410,13 @@ class _MachO(
)
case {
"Offset": offset,
"Section": {"Value": s},
"Type": {"Value": "X86_64_RELOC_SIGNED" as kind},
"Section": {"Name": s},
"Type": {"Name": "X86_64_RELOC_SIGNED" as kind},
} | {
"Offset": offset,
"Symbol": {"Value": s},
"Symbol": {"Name": s},
"Type": {
"Value": "X86_64_RELOC_BRANCH" | "X86_64_RELOC_SIGNED" as kind
"Name": "X86_64_RELOC_BRANCH" | "X86_64_RELOC_SIGNED" as kind
},
}:
offset += base
@ -427,12 +427,12 @@ class _MachO(
)
case {
"Offset": offset,
"Section": {"Value": s},
"Type": {"Value": kind},
"Section": {"Name": s},
"Type": {"Name": kind},
} | {
"Offset": offset,
"Symbol": {"Value": s},
"Type": {"Value": kind},
"Symbol": {"Name": s},
"Type": {"Name": kind},
}:
offset += base
s = s.removeprefix(self.prefix)
@ -446,13 +446,12 @@ class _MachO(
def get_target(host: str) -> _COFF | _ELF | _MachO:
"""Build a _Target for the given host "triple" and options."""
if re.fullmatch(r"aarch64-apple-darwin.*", host):
args = ["-mcmodel=large"]
return _MachO(host, alignment=8, args=args, prefix="_")
return _MachO(host, alignment=8, prefix="_")
if re.fullmatch(r"aarch64-pc-windows-msvc", host):
args = ["-fms-runtime-lib=dll"]
return _COFF(host, alignment=8, args=args)
if re.fullmatch(r"aarch64-.*-linux-gnu", host):
args = ["-mcmodel=large"]
args = ["-fpic"]
return _ELF(host, alignment=8, args=args)
if re.fullmatch(r"i686-pc-windows-msvc", host):
args = ["-DPy_NO_ENABLE_SHARED"]
@ -463,5 +462,6 @@ def get_target(host: str) -> _COFF | _ELF | _MachO:
args = ["-fms-runtime-lib=dll"]
return _COFF(host, args=args)
if re.fullmatch(r"x86_64-.*-linux-gnu", host):
return _ELF(host)
args = ["-fpic"]
return _ELF(host, args=args)
raise ValueError(host)