Merge for issue #24915
This commit is contained in:
commit
36df60fe75
|
@ -11,6 +11,9 @@
|
|||
*.rej
|
||||
*.swp
|
||||
*~
|
||||
*.gc??
|
||||
*.profclang?
|
||||
*.profraw
|
||||
.gdb_history
|
||||
Doc/build/
|
||||
Doc/venv/
|
||||
|
|
|
@ -50,6 +50,9 @@ libpython*.so*
|
|||
*.pyd
|
||||
*.cover
|
||||
*~
|
||||
*.gc??
|
||||
*.profclang?
|
||||
*.profraw
|
||||
Lib/distutils/command/*.pdb
|
||||
Lib/lib2to3/*.pickle
|
||||
Lib/test/data/*
|
||||
|
|
|
@ -43,6 +43,11 @@ LDVERSION= @LDVERSION@
|
|||
HGVERSION= @HGVERSION@
|
||||
HGTAG= @HGTAG@
|
||||
HGBRANCH= @HGBRANCH@
|
||||
PGO_PROF_GEN_FLAG=@PGO_PROF_GEN_FLAG@
|
||||
PGO_PROF_USE_FLAG=@PGO_PROF_USE_FLAG@
|
||||
LLVM_PROF_MERGER=@LLVM_PROF_MERGER@
|
||||
LLVM_PROF_FILE=@LLVM_PROF_FILE@
|
||||
LLVM_PROF_ERR=@LLVM_PROF_ERR@
|
||||
|
||||
GNULD= @GNULD@
|
||||
|
||||
|
@ -226,8 +231,7 @@ TCLTK_INCLUDES= @TCLTK_INCLUDES@
|
|||
TCLTK_LIBS= @TCLTK_LIBS@
|
||||
|
||||
# The task to run while instrument when building the profile-opt target
|
||||
PROFILE_TASK= $(srcdir)/Tools/pybench/pybench.py -n 2 --with-gc --with-syscheck
|
||||
#PROFILE_TASK= $(srcdir)/Lib/test/regrtest.py
|
||||
PROFILE_TASK=-m test.regrtest >/dev/null 2>&1
|
||||
|
||||
# report files for gcov / lcov coverage report
|
||||
COVERAGE_INFO= $(abs_builddir)/coverage.info
|
||||
|
@ -477,27 +481,38 @@ LIBRARY_OBJS= \
|
|||
all: build_all
|
||||
build_all: $(BUILDPYTHON) oldsharedmods sharedmods gdbhooks Programs/_testembed python-config
|
||||
|
||||
# Compile a binary with gcc profile guided optimization.
|
||||
# Compile a binary with profile guided optimization.
|
||||
profile-opt:
|
||||
@if [ $(LLVM_PROF_ERR) == yes ]; then \
|
||||
echo "Error: Cannot perform PGO build because llvm-profdata was not found in PATH" ;\
|
||||
echo "Please add it to PATH and run ./configure again" ;\
|
||||
exit 1;\
|
||||
fi
|
||||
@echo "Building with support for profile generation:"
|
||||
$(MAKE) clean
|
||||
$(MAKE) build_all_generate_profile
|
||||
@echo "Running benchmark to generate profile data:"
|
||||
$(MAKE) profile-removal
|
||||
$(MAKE) build_all_generate_profile
|
||||
$(MAKE) profile-removal
|
||||
@echo "Running code to generate profile data (this can take a while):"
|
||||
$(MAKE) run_profile_task
|
||||
$(MAKE) build_all_merge_profile
|
||||
@echo "Rebuilding with profile guided optimizations:"
|
||||
$(MAKE) clean
|
||||
$(MAKE) build_all_use_profile
|
||||
$(MAKE) profile-removal
|
||||
|
||||
build_all_generate_profile:
|
||||
$(MAKE) all CFLAGS_NODIST="$(CFLAGS) -fprofile-generate" LDFLAGS="-fprofile-generate" LIBS="$(LIBS) -lgcov"
|
||||
$(MAKE) all CFLAGS_NODIST="$(CFLAGS) $(PGO_PROF_GEN_FLAG)" LDFLAGS="$(LDFLAGS) $(PGO_PROF_GEN_FLAG)" LIBS="$(LIBS)"
|
||||
|
||||
run_profile_task:
|
||||
: # FIXME: can't run for a cross build
|
||||
$(RUNSHARED) ./$(BUILDPYTHON) $(PROFILE_TASK)
|
||||
$(LLVM_PROF_FILE) $(RUNSHARED) ./$(BUILDPYTHON) $(PROFILE_TASK) || true
|
||||
|
||||
build_all_merge_profile:
|
||||
$(LLVM_PROF_MERGER)
|
||||
|
||||
build_all_use_profile:
|
||||
$(MAKE) all CFLAGS_NODIST="$(CFLAGS) -fprofile-use -fprofile-correction"
|
||||
$(MAKE) all CFLAGS_NODIST="$(CFLAGS) $(PGO_PROF_USE_FLAG)"
|
||||
|
||||
# Compile and run with gcov
|
||||
.PHONY=coverage coverage-lcov coverage-report
|
||||
|
@ -1568,9 +1583,11 @@ clean: pycremoval
|
|||
-rm -f pybuilddir.txt
|
||||
-rm -f Lib/lib2to3/*Grammar*.pickle
|
||||
-rm -f Programs/_testembed Programs/_freeze_importlib
|
||||
-rm -rf build
|
||||
|
||||
profile-removal:
|
||||
find . -name '*.gc??' -exec rm -f {} ';'
|
||||
find . -name '*.profclang?' -exec rm -f {} ';'
|
||||
rm -f $(COVERAGE_INFO)
|
||||
rm -rf $(COVERAGE_REPORT)
|
||||
|
||||
|
|
|
@ -1080,6 +1080,7 @@ Heikki Partanen
|
|||
Harri Pasanen
|
||||
Gaël Pasgrimaud
|
||||
Ashish Nitin Patil
|
||||
Alecsandru Patrascu
|
||||
Randy Pausch
|
||||
Samuele Pedroni
|
||||
Justin Peel
|
||||
|
|
|
@ -201,6 +201,9 @@ Tests
|
|||
Build
|
||||
-----
|
||||
|
||||
- Issue #24915: Add LLVM support for PGO builds and use the test suite to
|
||||
generate the profile data. Initiial patch by Alecsandru Patrascu of Intel.
|
||||
|
||||
- Issue #24910: Windows MSIs now have unique display names.
|
||||
|
||||
- Issue #24986: It is now possible to build Python on Windows without errors
|
||||
|
|
28
README
28
README
|
@ -46,6 +46,34 @@ For example:
|
|||
(This will fail if you *also* built at the top-level directory.
|
||||
You should do a "make clean" at the toplevel first.)
|
||||
|
||||
If you need an optimized version of Python, you type "make profile-opt" in the
|
||||
top level directory. This will rebuild the interpreter executable using Profile
|
||||
Guided Optimization (PGO). For more details, see the section bellow.
|
||||
|
||||
|
||||
Profile Guided Optimization
|
||||
---------------------------
|
||||
|
||||
PGO takes advantage of recent versions of the GCC or Clang compilers.
|
||||
If ran, the "profile-opt" rule will do several steps.
|
||||
|
||||
First, the entire Python directory is cleaned of temporary files that
|
||||
may resulted in a previous compilation.
|
||||
|
||||
Then, an instrumented version of the interpreter is built, using suitable
|
||||
compiler flags for each flavour. Note that this is just an intermediary
|
||||
step and the binary resulted after this step is not good for real life
|
||||
workloads, as it has profiling instructions embedded inside.
|
||||
|
||||
After this instrumented version of the interpreter is built, the Makefile
|
||||
will automatically run a training workload. This is necessary in order to
|
||||
profile the interpreter execution. Note also that any output, both stdout
|
||||
and stderr, that may appear at this step is supressed.
|
||||
|
||||
Finally, the last step is to rebuild the interpreter, using the information
|
||||
collected in the previous one. The end result will be a the Python binary
|
||||
that is optimized and suitable for distribution or production installation.
|
||||
|
||||
|
||||
What's New
|
||||
----------
|
||||
|
|
|
@ -667,6 +667,12 @@ UNIVERSAL_ARCH_FLAGS
|
|||
CFLAGS_NODIST
|
||||
BASECFLAGS
|
||||
OPT
|
||||
LLVM_PROF_FOUND
|
||||
LLVM_PROF_ERR
|
||||
LLVM_PROF_FILE
|
||||
LLVM_PROF_MERGER
|
||||
PGO_PROF_USE_FLAG
|
||||
PGO_PROF_GEN_FLAG
|
||||
ABIFLAGS
|
||||
LN
|
||||
MKDIR_P
|
||||
|
@ -6431,6 +6437,80 @@ $as_echo "no" >&6; }
|
|||
fi
|
||||
|
||||
|
||||
# Enable PGO flags.
|
||||
# Extract the first word of "llvm-profdata", so it can be a program name with args.
|
||||
set dummy llvm-profdata; ac_word=$2
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
|
||||
$as_echo_n "checking for $ac_word... " >&6; }
|
||||
if ${ac_cv_prog_LLVM_PROF_FOUND+:} false; then :
|
||||
$as_echo_n "(cached) " >&6
|
||||
else
|
||||
if test -n "$LLVM_PROF_FOUND"; then
|
||||
ac_cv_prog_LLVM_PROF_FOUND="$LLVM_PROF_FOUND" # Let the user override the test.
|
||||
else
|
||||
as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
|
||||
for as_dir in $PATH
|
||||
do
|
||||
IFS=$as_save_IFS
|
||||
test -z "$as_dir" && as_dir=.
|
||||
for ac_exec_ext in '' $ac_executable_extensions; do
|
||||
if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
|
||||
ac_cv_prog_LLVM_PROF_FOUND="found"
|
||||
$as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
|
||||
break 2
|
||||
fi
|
||||
done
|
||||
done
|
||||
IFS=$as_save_IFS
|
||||
|
||||
test -z "$ac_cv_prog_LLVM_PROF_FOUND" && ac_cv_prog_LLVM_PROF_FOUND="not-found"
|
||||
fi
|
||||
fi
|
||||
LLVM_PROF_FOUND=$ac_cv_prog_LLVM_PROF_FOUND
|
||||
if test -n "$LLVM_PROF_FOUND"; then
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $LLVM_PROF_FOUND" >&5
|
||||
$as_echo "$LLVM_PROF_FOUND" >&6; }
|
||||
else
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
|
||||
$as_echo "no" >&6; }
|
||||
fi
|
||||
|
||||
|
||||
LLVM_PROF_ERR=no
|
||||
case $CC in
|
||||
*clang*)
|
||||
# Any changes made here should be reflected in the GCC+Darwin case below
|
||||
PGO_PROF_GEN_FLAG="-fprofile-instr-generate"
|
||||
PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd"
|
||||
LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr"
|
||||
LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\""
|
||||
if test $LLVM_PROF_FOUND = not-found
|
||||
then
|
||||
LLVM_PROF_ERR=yes
|
||||
fi
|
||||
;;
|
||||
*gcc*)
|
||||
case $ac_sys_system in
|
||||
Darwin*)
|
||||
PGO_PROF_GEN_FLAG="-fprofile-instr-generate"
|
||||
PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd"
|
||||
LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr"
|
||||
LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\""
|
||||
if test $LLVM_PROF_FOUND = not-found
|
||||
then
|
||||
LLVM_PROF_ERR=yes
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
PGO_PROF_GEN_FLAG="-fprofile-generate"
|
||||
PGO_PROF_USE_FLAG="-fprofile-use -fprofile-correction"
|
||||
LLVM_PROF_MERGER="true"
|
||||
LLVM_PROF_FILE=""
|
||||
;;
|
||||
esac
|
||||
;;
|
||||
esac
|
||||
|
||||
# XXX Shouldn't the code above that fiddles with BASECFLAGS and OPT be
|
||||
# merged with this chunk of code?
|
||||
|
||||
|
|
43
configure.ac
43
configure.ac
|
@ -1218,6 +1218,49 @@ else AC_MSG_RESULT(no); Py_DEBUG='false'
|
|||
fi],
|
||||
[AC_MSG_RESULT(no)])
|
||||
|
||||
# Enable PGO flags.
|
||||
AC_SUBST(PGO_PROF_GEN_FLAG)
|
||||
AC_SUBST(PGO_PROF_USE_FLAG)
|
||||
AC_SUBST(LLVM_PROF_MERGER)
|
||||
AC_SUBST(LLVM_PROF_FILE)
|
||||
AC_SUBST(LLVM_PROF_ERR)
|
||||
AC_SUBST(LLVM_PROF_FOUND)
|
||||
AC_CHECK_PROG(LLVM_PROF_FOUND, llvm-profdata, found, not-found)
|
||||
LLVM_PROF_ERR=no
|
||||
case $CC in
|
||||
*clang*)
|
||||
# Any changes made here should be reflected in the GCC+Darwin case below
|
||||
PGO_PROF_GEN_FLAG="-fprofile-instr-generate"
|
||||
PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd"
|
||||
LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr"
|
||||
LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\""
|
||||
if test $LLVM_PROF_FOUND = not-found
|
||||
then
|
||||
LLVM_PROF_ERR=yes
|
||||
fi
|
||||
;;
|
||||
*gcc*)
|
||||
case $ac_sys_system in
|
||||
Darwin*)
|
||||
PGO_PROF_GEN_FLAG="-fprofile-instr-generate"
|
||||
PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd"
|
||||
LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr"
|
||||
LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\""
|
||||
if test $LLVM_PROF_FOUND = not-found
|
||||
then
|
||||
LLVM_PROF_ERR=yes
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
PGO_PROF_GEN_FLAG="-fprofile-generate"
|
||||
PGO_PROF_USE_FLAG="-fprofile-use -fprofile-correction"
|
||||
LLVM_PROF_MERGER="true"
|
||||
LLVM_PROF_FILE=""
|
||||
;;
|
||||
esac
|
||||
;;
|
||||
esac
|
||||
|
||||
# XXX Shouldn't the code above that fiddles with BASECFLAGS and OPT be
|
||||
# merged with this chunk of code?
|
||||
|
||||
|
|
Loading…
Reference in New Issue