From 7188a3efe07b9effdb760f3a96783f250214f0be Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 18 Sep 2015 15:13:44 -0700 Subject: [PATCH] Issue #24915: Add Clang support to PGO builds and use the test suite for profile data. Thanks to Alecsandru Patrascu of Intel for the initial patch. --- .gitignore | 3 ++ .hgignore | 3 ++ Makefile.pre.in | 33 +++++++++++++++----- Misc/ACKS | 1 + Misc/NEWS | 3 ++ README | 28 +++++++++++++++++ configure | 80 +++++++++++++++++++++++++++++++++++++++++++++++++ configure.ac | 43 ++++++++++++++++++++++++++ 8 files changed, 186 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index cc1cdcd5052..5be9ae80dc7 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,9 @@ *.rej *.swp *~ +*.gc?? +*.profclang? +*.profraw .gdb_history Doc/build/ Doc/venv/ diff --git a/.hgignore b/.hgignore index 806e15606ec..7f14dd63111 100644 --- a/.hgignore +++ b/.hgignore @@ -50,6 +50,9 @@ libpython*.so* *.pyd *.cover *~ +*.gc?? +*.profclang? +*.profraw Lib/distutils/command/*.pdb Lib/lib2to3/*.pickle Lib/test/data/* diff --git a/Makefile.pre.in b/Makefile.pre.in index ce2c0aa2900..9a8c1599b99 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -43,6 +43,11 @@ LDVERSION= @LDVERSION@ HGVERSION= @HGVERSION@ HGTAG= @HGTAG@ HGBRANCH= @HGBRANCH@ +PGO_PROF_GEN_FLAG=@PGO_PROF_GEN_FLAG@ +PGO_PROF_USE_FLAG=@PGO_PROF_USE_FLAG@ +LLVM_PROF_MERGER=@LLVM_PROF_MERGER@ +LLVM_PROF_FILE=@LLVM_PROF_FILE@ +LLVM_PROF_ERR=@LLVM_PROF_ERR@ GNULD= @GNULD@ @@ -226,8 +231,7 @@ TCLTK_INCLUDES= @TCLTK_INCLUDES@ TCLTK_LIBS= @TCLTK_LIBS@ # The task to run while instrument when building the profile-opt target -PROFILE_TASK= $(srcdir)/Tools/pybench/pybench.py -n 2 --with-gc --with-syscheck -#PROFILE_TASK= $(srcdir)/Lib/test/regrtest.py +PROFILE_TASK=-m test.regrtest >/dev/null 2>&1 # report files for gcov / lcov coverage report COVERAGE_INFO= $(abs_builddir)/coverage.info @@ -477,27 +481,38 @@ LIBRARY_OBJS= \ all: build_all build_all: $(BUILDPYTHON) oldsharedmods sharedmods gdbhooks Programs/_testembed python-config -# Compile a binary with gcc profile guided optimization. +# Compile a binary with profile guided optimization. profile-opt: + @if [ $(LLVM_PROF_ERR) == yes ]; then \ + echo "Error: Cannot perform PGO build because llvm-profdata was not found in PATH" ;\ + echo "Please add it to PATH and run ./configure again" ;\ + exit 1;\ + fi @echo "Building with support for profile generation:" $(MAKE) clean - $(MAKE) build_all_generate_profile - @echo "Running benchmark to generate profile data:" $(MAKE) profile-removal + $(MAKE) build_all_generate_profile + $(MAKE) profile-removal + @echo "Running code to generate profile data (this can take a while):" $(MAKE) run_profile_task + $(MAKE) build_all_merge_profile @echo "Rebuilding with profile guided optimizations:" $(MAKE) clean $(MAKE) build_all_use_profile + $(MAKE) profile-removal build_all_generate_profile: - $(MAKE) all CFLAGS_NODIST="$(CFLAGS) -fprofile-generate" LDFLAGS="-fprofile-generate" LIBS="$(LIBS) -lgcov" + $(MAKE) all CFLAGS_NODIST="$(CFLAGS) $(PGO_PROF_GEN_FLAG)" LDFLAGS="$(LDFLAGS) $(PGO_PROF_GEN_FLAG)" LIBS="$(LIBS)" run_profile_task: : # FIXME: can't run for a cross build - $(RUNSHARED) ./$(BUILDPYTHON) $(PROFILE_TASK) + $(LLVM_PROF_FILE) $(RUNSHARED) ./$(BUILDPYTHON) $(PROFILE_TASK) || true + +build_all_merge_profile: + $(LLVM_PROF_MERGER) build_all_use_profile: - $(MAKE) all CFLAGS_NODIST="$(CFLAGS) -fprofile-use -fprofile-correction" + $(MAKE) all CFLAGS_NODIST="$(CFLAGS) $(PGO_PROF_USE_FLAG)" # Compile and run with gcov .PHONY=coverage coverage-lcov coverage-report @@ -1568,9 +1583,11 @@ clean: pycremoval -rm -f pybuilddir.txt -rm -f Lib/lib2to3/*Grammar*.pickle -rm -f Programs/_testembed Programs/_freeze_importlib + -rm -rf build profile-removal: find . -name '*.gc??' -exec rm -f {} ';' + find . -name '*.profclang?' -exec rm -f {} ';' rm -f $(COVERAGE_INFO) rm -rf $(COVERAGE_REPORT) diff --git a/Misc/ACKS b/Misc/ACKS index 3704e650bfe..ba144b70329 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1079,6 +1079,7 @@ Heikki Partanen Harri Pasanen Gaël Pasgrimaud Ashish Nitin Patil +Alecsandru Patrascu Randy Pausch Samuele Pedroni Justin Peel diff --git a/Misc/NEWS b/Misc/NEWS index c021452c502..edd555feeba 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -115,6 +115,9 @@ Tests Build ----- +- Issue #24915: Add LLVM support for PGO builds and use the test suite to + generate the profile data. Initiial patch by Alecsandru Patrascu of Intel. + - Issue #24910: Windows MSIs now have unique display names. - Issue #24986: It is now possible to build Python on Windows without errors diff --git a/README b/README index d37b96b3160..82d1697accf 100644 --- a/README +++ b/README @@ -46,6 +46,34 @@ For example: (This will fail if you *also* built at the top-level directory. You should do a "make clean" at the toplevel first.) +If you need an optimized version of Python, you type "make profile-opt" in the +top level directory. This will rebuild the interpreter executable using Profile +Guided Optimization (PGO). For more details, see the section bellow. + + +Profile Guided Optimization +--------------------------- + +PGO takes advantage of recent versions of the GCC or Clang compilers. +If ran, the "profile-opt" rule will do several steps. + +First, the entire Python directory is cleaned of temporary files that +may resulted in a previous compilation. + +Then, an instrumented version of the interpreter is built, using suitable +compiler flags for each flavour. Note that this is just an intermediary +step and the binary resulted after this step is not good for real life +workloads, as it has profiling instructions embedded inside. + +After this instrumented version of the interpreter is built, the Makefile +will automatically run a training workload. This is necessary in order to +profile the interpreter execution. Note also that any output, both stdout +and stderr, that may appear at this step is supressed. + +Finally, the last step is to rebuild the interpreter, using the information +collected in the previous one. The end result will be a the Python binary +that is optimized and suitable for distribution or production installation. + What's New ---------- diff --git a/configure b/configure index e823a083c67..06b4d582e09 100755 --- a/configure +++ b/configure @@ -667,6 +667,12 @@ UNIVERSAL_ARCH_FLAGS CFLAGS_NODIST BASECFLAGS OPT +LLVM_PROF_FOUND +LLVM_PROF_ERR +LLVM_PROF_FILE +LLVM_PROF_MERGER +PGO_PROF_USE_FLAG +PGO_PROF_GEN_FLAG ABIFLAGS LN MKDIR_P @@ -6431,6 +6437,80 @@ $as_echo "no" >&6; } fi +# Enable PGO flags. +# Extract the first word of "llvm-profdata", so it can be a program name with args. +set dummy llvm-profdata; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_LLVM_PROF_FOUND+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$LLVM_PROF_FOUND"; then + ac_cv_prog_LLVM_PROF_FOUND="$LLVM_PROF_FOUND" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_prog_LLVM_PROF_FOUND="found" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + test -z "$ac_cv_prog_LLVM_PROF_FOUND" && ac_cv_prog_LLVM_PROF_FOUND="not-found" +fi +fi +LLVM_PROF_FOUND=$ac_cv_prog_LLVM_PROF_FOUND +if test -n "$LLVM_PROF_FOUND"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $LLVM_PROF_FOUND" >&5 +$as_echo "$LLVM_PROF_FOUND" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + +LLVM_PROF_ERR=no +case $CC in + *clang*) + # Any changes made here should be reflected in the GCC+Darwin case below + PGO_PROF_GEN_FLAG="-fprofile-instr-generate" + PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd" + LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr" + LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\"" + if test $LLVM_PROF_FOUND = not-found + then + LLVM_PROF_ERR=yes + fi + ;; + *gcc*) + case $ac_sys_system in + Darwin*) + PGO_PROF_GEN_FLAG="-fprofile-instr-generate" + PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd" + LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr" + LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\"" + if test $LLVM_PROF_FOUND = not-found + then + LLVM_PROF_ERR=yes + fi + ;; + *) + PGO_PROF_GEN_FLAG="-fprofile-generate" + PGO_PROF_USE_FLAG="-fprofile-use -fprofile-correction" + LLVM_PROF_MERGER="true" + LLVM_PROF_FILE="" + ;; + esac + ;; +esac + # XXX Shouldn't the code above that fiddles with BASECFLAGS and OPT be # merged with this chunk of code? diff --git a/configure.ac b/configure.ac index 56a73df233d..792f580c12a 100644 --- a/configure.ac +++ b/configure.ac @@ -1218,6 +1218,49 @@ else AC_MSG_RESULT(no); Py_DEBUG='false' fi], [AC_MSG_RESULT(no)]) +# Enable PGO flags. +AC_SUBST(PGO_PROF_GEN_FLAG) +AC_SUBST(PGO_PROF_USE_FLAG) +AC_SUBST(LLVM_PROF_MERGER) +AC_SUBST(LLVM_PROF_FILE) +AC_SUBST(LLVM_PROF_ERR) +AC_SUBST(LLVM_PROF_FOUND) +AC_CHECK_PROG(LLVM_PROF_FOUND, llvm-profdata, found, not-found) +LLVM_PROF_ERR=no +case $CC in + *clang*) + # Any changes made here should be reflected in the GCC+Darwin case below + PGO_PROF_GEN_FLAG="-fprofile-instr-generate" + PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd" + LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr" + LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\"" + if test $LLVM_PROF_FOUND = not-found + then + LLVM_PROF_ERR=yes + fi + ;; + *gcc*) + case $ac_sys_system in + Darwin*) + PGO_PROF_GEN_FLAG="-fprofile-instr-generate" + PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd" + LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr" + LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\"" + if test $LLVM_PROF_FOUND = not-found + then + LLVM_PROF_ERR=yes + fi + ;; + *) + PGO_PROF_GEN_FLAG="-fprofile-generate" + PGO_PROF_USE_FLAG="-fprofile-use -fprofile-correction" + LLVM_PROF_MERGER="true" + LLVM_PROF_FILE="" + ;; + esac + ;; +esac + # XXX Shouldn't the code above that fiddles with BASECFLAGS and OPT be # merged with this chunk of code?