diff -r f5c40ab9e233 .gitignore --- a/.gitignore Tue Aug 18 22:25:16 2015 -0700 +++ b/.gitignore Mon Sep 07 15:04:21 2015 +0300 @@ -11,6 +11,9 @@ *.rej *.swp *~ +*.gc?? +*.profclang? +*.profraw .gdb_history Doc/build/ Doc/venv/ diff -r f5c40ab9e233 .hgignore --- a/.hgignore Tue Aug 18 22:25:16 2015 -0700 +++ b/.hgignore Mon Sep 07 15:04:21 2015 +0300 @@ -50,6 +50,9 @@ *.pyd *.cover *~ +*.gc?? +*.profclang? +*.profraw Lib/distutils/command/*.pdb Lib/lib2to3/*.pickle Lib/test/data/* diff -r f5c40ab9e233 Makefile.pre.in --- a/Makefile.pre.in Tue Aug 18 22:25:16 2015 -0700 +++ b/Makefile.pre.in Mon Sep 07 15:04:21 2015 +0300 @@ -43,6 +43,11 @@ HGVERSION= @HGVERSION@ HGTAG= @HGTAG@ HGBRANCH= @HGBRANCH@ +PGO_PROF_GEN_FLAG=@PGO_PROF_GEN_FLAG@ +PGO_PROF_USE_FLAG=@PGO_PROF_USE_FLAG@ +LLVM_PROF_MERGER=@LLVM_PROF_MERGER@ +LLVM_PROF_FILE=@LLVM_PROF_FILE@ +LLVM_PROF_ERR=@LLVM_PROF_ERR@ GNULD= @GNULD@ @@ -226,8 +231,8 @@ TCLTK_LIBS= @TCLTK_LIBS@ # The task to run while instrument when building the profile-opt target -PROFILE_TASK= $(srcdir)/Tools/pybench/pybench.py -n 2 --with-gc --with-syscheck -#PROFILE_TASK= $(srcdir)/Lib/test/regrtest.py +PROFILE_TASK=-m test.regrtest >/dev/null 2>&1 +PROFILE_TASK_OLD=$(srcdir)/Tools/pybench/pybench.py -n 2 --with-gc --with-syscheck # report files for gcov / lcov coverage report COVERAGE_INFO= $(abs_builddir)/coverage.info @@ -477,27 +482,47 @@ all: build_all build_all: $(BUILDPYTHON) oldsharedmods sharedmods gdbhooks Programs/_testembed python-config -# Compile a binary with gcc profile guided optimization. +# Compile a binary with profile guided optimization. profile-opt: + @if [ $(LLVM_PROF_ERR) == yes ]; then \ + echo "Error: Cannot perform PGO build because llvm-profdata was not found in PATH" ;\ + echo "Please add it to PATH and run ./configure again" ;\ + exit 1; \ + fi + @if [ `uname -m | grep 86` == '' ];then \ + echo "Warning: the benefits of profile-guided optimization only tested x86 architectures!" ;\ + fi @echo "Building with support for profile generation:" $(MAKE) clean + $(MAKE) profile-removal $(MAKE) build_all_generate_profile - @echo "Running benchmark to generate profile data:" $(MAKE) profile-removal + @echo "Running code to generate profile data:" + @echo "Warning: all output is suppressed. Please wait until the profile is generated." + @echo "Code failures will not adversely affect the results!" $(MAKE) run_profile_task + $(MAKE) build_all_merge_profile @echo "Rebuilding with profile guided optimizations:" $(MAKE) clean $(MAKE) build_all_use_profile + $(MAKE) profile-removal build_all_generate_profile: - $(MAKE) all CFLAGS_NODIST="$(CFLAGS) -fprofile-generate" LDFLAGS="-fprofile-generate" LIBS="$(LIBS) -lgcov" + $(MAKE) all CFLAGS_NODIST="$(CFLAGS) $(PGO_PROF_GEN_FLAG)" LDFLAGS="$(LDFLAGS) $(PGO_PROF_GEN_FLAG)" LIBS="$(LIBS)" run_profile_task: : # FIXME: can't run for a cross build - $(RUNSHARED) ./$(BUILDPYTHON) $(PROFILE_TASK) + @if [ `uname -m | grep 86` == '' ];then \ + $(LLVM_PROF_FILE) $(RUNSHARED) ./$(BUILDPYTHON) $(PROFILE_TASK_OLD) || true ;\ + else \ + $(LLVM_PROF_FILE) $(RUNSHARED) ./$(BUILDPYTHON) $(PROFILE_TASK) || true ;\ + fi + +build_all_merge_profile: + $(LLVM_PROF_MERGER) build_all_use_profile: - $(MAKE) all CFLAGS_NODIST="$(CFLAGS) -fprofile-use -fprofile-correction" + $(MAKE) all CFLAGS_NODIST="$(CFLAGS) $(PGO_PROF_USE_FLAG)" # Compile and run with gcov .PHONY=coverage coverage-lcov coverage-report @@ -1568,9 +1593,11 @@ -rm -f pybuilddir.txt -rm -f Lib/lib2to3/*Grammar*.pickle -rm -f Programs/_testembed Programs/_freeze_importlib + -rm -rf build profile-removal: find . -name '*.gc??' -exec rm -f {} ';' + find . -name '*.profclang?' -exec rm -f {} ';' rm -f $(COVERAGE_INFO) rm -rf $(COVERAGE_REPORT) diff -r f5c40ab9e233 configure --- a/configure Tue Aug 18 22:25:16 2015 -0700 +++ b/configure Mon Sep 07 15:04:21 2015 +0300 @@ -667,6 +667,12 @@ CFLAGS_NODIST BASECFLAGS OPT +LLVM_PROF_FOUND +LLVM_PROF_ERR +LLVM_PROF_FILE +LLVM_PROF_MERGER +PGO_PROF_USE_FLAG +PGO_PROF_GEN_FLAG ABIFLAGS LN MKDIR_P @@ -6431,6 +6437,80 @@ fi +# Enable PGO flags. +# Extract the first word of "llvm-profdata", so it can be a program name with args. +set dummy llvm-profdata; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_LLVM_PROF_FOUND+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$LLVM_PROF_FOUND"; then + ac_cv_prog_LLVM_PROF_FOUND="$LLVM_PROF_FOUND" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_prog_LLVM_PROF_FOUND="found" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + test -z "$ac_cv_prog_LLVM_PROF_FOUND" && ac_cv_prog_LLVM_PROF_FOUND="not-found" +fi +fi +LLVM_PROF_FOUND=$ac_cv_prog_LLVM_PROF_FOUND +if test -n "$LLVM_PROF_FOUND"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $LLVM_PROF_FOUND" >&5 +$as_echo "$LLVM_PROF_FOUND" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + +LLVM_PROF_ERR=no +case $CC in + *clang*) + # Any changes made here should be reflected in the GCC+Darwin case below + PGO_PROF_GEN_FLAG="-fprofile-instr-generate" + PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd" + LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr" + LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\"" + if test $LLVM_PROF_FOUND = not-found + then + LLVM_PROF_ERR=yes + fi + ;; + *gcc*) + case $ac_sys_system in + Darwin*) + PGO_PROF_GEN_FLAG="-fprofile-instr-generate" + PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd" + LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr" + LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\"" + if test $LLVM_PROF_FOUND = not-found + then + LLVM_PROF_ERR=yes + fi + ;; + *) + PGO_PROF_GEN_FLAG="-fprofile-generate" + PGO_PROF_USE_FLAG="-fprofile-use -fprofile-correction" + LLVM_PROF_MERGER="true" + LLVM_PROF_FILE="" + ;; + esac + ;; +esac + # XXX Shouldn't the code above that fiddles with BASECFLAGS and OPT be # merged with this chunk of code? diff -r f5c40ab9e233 configure.ac --- a/configure.ac Tue Aug 18 22:25:16 2015 -0700 +++ b/configure.ac Mon Sep 07 15:04:21 2015 +0300 @@ -1218,6 +1218,49 @@ fi], [AC_MSG_RESULT(no)]) +# Enable PGO flags. +AC_SUBST(PGO_PROF_GEN_FLAG) +AC_SUBST(PGO_PROF_USE_FLAG) +AC_SUBST(LLVM_PROF_MERGER) +AC_SUBST(LLVM_PROF_FILE) +AC_SUBST(LLVM_PROF_ERR) +AC_SUBST(LLVM_PROF_FOUND) +AC_CHECK_PROG(LLVM_PROF_FOUND, llvm-profdata, found, not-found) +LLVM_PROF_ERR=no +case $CC in + *clang*) + # Any changes made here should be reflected in the GCC+Darwin case below + PGO_PROF_GEN_FLAG="-fprofile-instr-generate" + PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd" + LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr" + LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\"" + if test $LLVM_PROF_FOUND = not-found + then + LLVM_PROF_ERR=yes + fi + ;; + *gcc*) + case $ac_sys_system in + Darwin*) + PGO_PROF_GEN_FLAG="-fprofile-instr-generate" + PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd" + LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr" + LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\"" + if test $LLVM_PROF_FOUND = not-found + then + LLVM_PROF_ERR=yes + fi + ;; + *) + PGO_PROF_GEN_FLAG="-fprofile-generate" + PGO_PROF_USE_FLAG="-fprofile-use -fprofile-correction" + LLVM_PROF_MERGER="true" + LLVM_PROF_FILE="" + ;; + esac + ;; +esac + # XXX Shouldn't the code above that fiddles with BASECFLAGS and OPT be # merged with this chunk of code?