diff -r da483bda6efc .gitignore --- a/.gitignore Mon Aug 24 16:17:13 2015 -0400 +++ b/.gitignore Mon Sep 07 15:09:01 2015 +0300 @@ -6,6 +6,9 @@ *.pyo *.rej *~ +*.gc?? +*.profclang? +*.profraw Doc/build/ Doc/tools/docutils/ Doc/tools/jinja2/ diff -r da483bda6efc .hgignore --- a/.hgignore Mon Aug 24 16:17:13 2015 -0400 +++ b/.hgignore Mon Sep 07 15:09:01 2015 +0300 @@ -45,6 +45,9 @@ *.pyd *.cover *~ +*.gc?? +*.profclang? +*.profraw Lib/distutils/command/*.pdb Lib/lib2to3/*.pickle Lib/test/data/* diff -r da483bda6efc Makefile.pre.in --- a/Makefile.pre.in Mon Aug 24 16:17:13 2015 -0400 +++ b/Makefile.pre.in Mon Sep 07 15:09:01 2015 +0300 @@ -42,6 +42,11 @@ HGVERSION= @HGVERSION@ HGTAG= @HGTAG@ HGBRANCH= @HGBRANCH@ +PGO_PROF_GEN_FLAG=@PGO_PROF_GEN_FLAG@ +PGO_PROF_USE_FLAG=@PGO_PROF_USE_FLAG@ +LLVM_PROF_MERGER=@LLVM_PROF_MERGER@ +LLVM_PROF_FILE=@LLVM_PROF_FILE@ +LLVM_PROF_ERR=@LLVM_PROF_ERR@ GNULD= @GNULD@ @@ -204,8 +209,8 @@ TCLTK_LIBS= @TCLTK_LIBS@ # The task to run while instrument when building the profile-opt target -PROFILE_TASK= $(srcdir)/Tools/pybench/pybench.py -n 2 --with-gc --with-syscheck -#PROFILE_TASK= $(srcdir)/Lib/test/regrtest.py +PROFILE_TASK=-m test.regrtest >/dev/null 2>&1 +PROFILE_TASK_OLD=$(srcdir)/Tools/pybench/pybench.py -n 2 --with-gc --with-syscheck # === Definitions added by makesetup === @@ -420,27 +425,47 @@ all: build_all build_all: $(BUILDPYTHON) oldsharedmods sharedmods gdbhooks -# Compile a binary with gcc profile guided optimization. +# Compile a binary with profile guided optimization. profile-opt: + @if [ $(LLVM_PROF_ERR) == yes ]; then \ + echo "Error: Cannot perform PGO build because llvm-profdata was not found in PATH" ;\ + echo "Please add it to PATH and run ./configure again" ;\ + exit 1;\ + fi + @if [ `uname -m | grep 86` == '' ];then \ + echo "Warning: the benefits of profile-guided optimization only tested x86 architectures!" ;\ + fi @echo "Building with support for profile generation:" $(MAKE) clean + $(MAKE) profile-removal $(MAKE) build_all_generate_profile - @echo "Running benchmark to generate profile data:" $(MAKE) profile-removal + @echo "Running code to generate profile data:" + @echo "Warning: all output is suppressed. Please wait until the profile is generated." + @echo "Code failures will not adversely affect the results!" $(MAKE) run_profile_task + $(MAKE) build_all_merge_profile @echo "Rebuilding with profile guided optimizations:" $(MAKE) clean $(MAKE) build_all_use_profile + $(MAKE) profile-removal build_all_generate_profile: - $(MAKE) all CFLAGS="$(CFLAGS) -fprofile-generate" LIBS="$(LIBS) -lgcov" + $(MAKE) all CFLAGS="$(CFLAGS) $(PGO_PROF_GEN_FLAG)" LDFLAGS="$(LDFLAGS) $(PGO_PROF_GEN_FLAG)" LIBS="$(LIBS)" run_profile_task: : # FIXME: can't run for a cross build - ./$(BUILDPYTHON) $(PROFILE_TASK) + @if [ `uname -m | grep 86` == '' ];then \ + $(LLVM_PROF_FILE) ./$(BUILDPYTHON) $(PROFILE_TASK_OLD) || true ;\ + else \ + $(LLVM_PROF_FILE) ./$(BUILDPYTHON) $(PROFILE_TASK) || true ;\ + fi + +build_all_merge_profile: + $(LLVM_PROF_MERGER) build_all_use_profile: - $(MAKE) all CFLAGS="$(CFLAGS) -fprofile-use" + $(MAKE) all CFLAGS="$(CFLAGS) $(PGO_PROF_USE_FLAG)" coverage: @echo "Building with support for coverage checking:" @@ -1330,9 +1355,11 @@ find build -name 'fficonfig.h' -exec rm -f {} ';' || true find build -name 'fficonfig.py' -exec rm -f {} ';' || true -rm -f Lib/lib2to3/*Grammar*.pickle + -rm -rf build profile-removal: find . -name '*.gc??' -exec rm -f {} ';' + find . -name '*.profclang?' -exec rm -f {} ';' clobber: clean profile-removal -rm -f $(BUILDPYTHON) $(PGEN) $(LIBRARY) $(LDLIBRARY) $(DLLLIBRARY) \ diff -r da483bda6efc configure --- a/configure Mon Aug 24 16:17:13 2015 -0400 +++ b/configure Mon Sep 07 15:09:01 2015 +0300 @@ -661,6 +661,12 @@ SO LIBTOOL_CRUFT OTHER_LIBTOOL_OPT +LLVM_PROF_FOUND +LLVM_PROF_ERR +LLVM_PROF_FILE +LLVM_PROF_MERGER +PGO_PROF_USE_FLAG +PGO_PROF_GEN_FLAG UNIVERSAL_ARCH_FLAGS BASECFLAGS OPT @@ -6326,6 +6332,82 @@ CFLAGS=$save_CFLAGS fi + +# Enable PGO flags. +# Extract the first word of "llvm-profdata", so it can be a program name with args. +set dummy llvm-profdata; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_prog_LLVM_PROF_FOUND+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test -n "$LLVM_PROF_FOUND"; then + ac_cv_prog_LLVM_PROF_FOUND="$LLVM_PROF_FOUND" # Let the user override the test. +else +as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_prog_LLVM_PROF_FOUND="found" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + test -z "$ac_cv_prog_LLVM_PROF_FOUND" && ac_cv_prog_LLVM_PROF_FOUND="not-found" +fi +fi +LLVM_PROF_FOUND=$ac_cv_prog_LLVM_PROF_FOUND +if test -n "$LLVM_PROF_FOUND"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $LLVM_PROF_FOUND" >&5 +$as_echo "$LLVM_PROF_FOUND" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + +LLVM_PROF_ERR=no +case $CC in + *clang*) + # Any changes made here should be reflected in the GCC+Darwin case below + PGO_PROF_GEN_FLAG="-fprofile-instr-generate" + PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd" + LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr" + LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\"" + if test $LLVM_PROF_FOUND = not-found + then + LLVM_PROF_ERR=yes + fi + ;; + *gcc*) + case $ac_sys_system in + Darwin*) + PGO_PROF_GEN_FLAG="-fprofile-instr-generate" + PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd" + LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr" + LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\"" + if test $LLVM_PROF_FOUND = not-found + then + LLVM_PROF_ERR=yes + fi + ;; + *) + PGO_PROF_GEN_FLAG="-fprofile-generate" + PGO_PROF_USE_FLAG="-fprofile-use -fprofile-correction" + LLVM_PROF_MERGER="true" + LLVM_PROF_FILE="" + ;; + esac + ;; +esac + + # On some compilers, pthreads are available without further options # (e.g. MacOS X). On some of these systems, the compiler will not # complain if unaccepted options are passed (e.g. gcc on Mac OS X). diff -r da483bda6efc configure.ac --- a/configure.ac Mon Aug 24 16:17:13 2015 -0400 +++ b/configure.ac Mon Sep 07 15:09:01 2015 +0300 @@ -1352,6 +1352,51 @@ CFLAGS=$save_CFLAGS fi + +# Enable PGO flags. +AC_SUBST(PGO_PROF_GEN_FLAG) +AC_SUBST(PGO_PROF_USE_FLAG) +AC_SUBST(LLVM_PROF_MERGER) +AC_SUBST(LLVM_PROF_FILE) +AC_SUBST(LLVM_PROF_ERR) +AC_SUBST(LLVM_PROF_FOUND) +AC_CHECK_PROG(LLVM_PROF_FOUND, llvm-profdata, found, not-found) +LLVM_PROF_ERR=no +case $CC in + *clang*) + # Any changes made here should be reflected in the GCC+Darwin case below + PGO_PROF_GEN_FLAG="-fprofile-instr-generate" + PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd" + LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr" + LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\"" + if test $LLVM_PROF_FOUND = not-found + then + LLVM_PROF_ERR=yes + fi + ;; + *gcc*) + case $ac_sys_system in + Darwin*) + PGO_PROF_GEN_FLAG="-fprofile-instr-generate" + PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd" + LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr" + LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\"" + if test $LLVM_PROF_FOUND = not-found + then + LLVM_PROF_ERR=yes + fi + ;; + *) + PGO_PROF_GEN_FLAG="-fprofile-generate" + PGO_PROF_USE_FLAG="-fprofile-use -fprofile-correction" + LLVM_PROF_MERGER="true" + LLVM_PROF_FILE="" + ;; + esac + ;; +esac + + # On some compilers, pthreads are available without further options # (e.g. MacOS X). On some of these systems, the compiler will not # complain if unaccepted options are passed (e.g. gcc on Mac OS X).