[Python-checkins] cpython (merge 3.5 -> default): Merge for issue #24915
brett.cannon
python-checkins at python.org
Sat Sep 19 00:17:44 CEST 2015
https://hg.python.org/cpython/rev/7749fc0a5ea6
changeset: 98062:7749fc0a5ea6
parent: 98058:97f853542dd6
parent: 98061:7fcff838d09e
user: Brett Cannon <brett at python.org>
date: Fri Sep 18 15:17:37 2015 -0700
summary:
Merge for issue #24915
files:
.gitignore | 3 +
.hgignore | 3 +
Makefile.pre.in | 31 +++++++++++---
Misc/ACKS | 1 +
Misc/NEWS | 3 +
README | 28 ++++++++++++
configure | 80 +++++++++++++++++++++++++++++++++++++
configure.ac | 43 +++++++++++++++++++
8 files changed, 185 insertions(+), 7 deletions(-)
diff --git a/.gitignore b/.gitignore
--- a/.gitignore
+++ b/.gitignore
@@ -11,6 +11,9 @@
*.rej
*.swp
*~
+*.gc??
+*.profclang?
+*.profraw
.gdb_history
Doc/build/
Doc/venv/
diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -50,6 +50,9 @@
*.pyd
*.cover
*~
+*.gc??
+*.profclang?
+*.profraw
Lib/distutils/command/*.pdb
Lib/lib2to3/*.pickle
Lib/test/data/*
diff --git a/Makefile.pre.in b/Makefile.pre.in
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -43,6 +43,11 @@
HGVERSION= @HGVERSION@
HGTAG= @HGTAG@
HGBRANCH= @HGBRANCH@
+PGO_PROF_GEN_FLAG=@PGO_PROF_GEN_FLAG@
+PGO_PROF_USE_FLAG=@PGO_PROF_USE_FLAG@
+LLVM_PROF_MERGER=@LLVM_PROF_MERGER@
+LLVM_PROF_FILE=@LLVM_PROF_FILE@
+LLVM_PROF_ERR=@LLVM_PROF_ERR@
GNULD= @GNULD@
@@ -226,8 +231,7 @@
TCLTK_LIBS= @TCLTK_LIBS@
# The task to run while instrument when building the profile-opt target
-PROFILE_TASK= $(srcdir)/Tools/pybench/pybench.py -n 2 --with-gc --with-syscheck
-#PROFILE_TASK= $(srcdir)/Lib/test/regrtest.py
+PROFILE_TASK=-m test.regrtest >/dev/null 2>&1
# report files for gcov / lcov coverage report
COVERAGE_INFO= $(abs_builddir)/coverage.info
@@ -477,27 +481,38 @@
all: build_all
build_all: $(BUILDPYTHON) oldsharedmods sharedmods gdbhooks Programs/_testembed python-config
-# Compile a binary with gcc profile guided optimization.
+# Compile a binary with profile guided optimization.
profile-opt:
+ @if [ $(LLVM_PROF_ERR) == yes ]; then \
+ echo "Error: Cannot perform PGO build because llvm-profdata was not found in PATH" ;\
+ echo "Please add it to PATH and run ./configure again" ;\
+ exit 1;\
+ fi
@echo "Building with support for profile generation:"
$(MAKE) clean
+ $(MAKE) profile-removal
$(MAKE) build_all_generate_profile
- @echo "Running benchmark to generate profile data:"
$(MAKE) profile-removal
+ @echo "Running code to generate profile data (this can take a while):"
$(MAKE) run_profile_task
+ $(MAKE) build_all_merge_profile
@echo "Rebuilding with profile guided optimizations:"
$(MAKE) clean
$(MAKE) build_all_use_profile
+ $(MAKE) profile-removal
build_all_generate_profile:
- $(MAKE) all CFLAGS_NODIST="$(CFLAGS) -fprofile-generate" LDFLAGS="-fprofile-generate" LIBS="$(LIBS) -lgcov"
+ $(MAKE) all CFLAGS_NODIST="$(CFLAGS) $(PGO_PROF_GEN_FLAG)" LDFLAGS="$(LDFLAGS) $(PGO_PROF_GEN_FLAG)" LIBS="$(LIBS)"
run_profile_task:
: # FIXME: can't run for a cross build
- $(RUNSHARED) ./$(BUILDPYTHON) $(PROFILE_TASK)
+ $(LLVM_PROF_FILE) $(RUNSHARED) ./$(BUILDPYTHON) $(PROFILE_TASK) || true
+
+build_all_merge_profile:
+ $(LLVM_PROF_MERGER)
build_all_use_profile:
- $(MAKE) all CFLAGS_NODIST="$(CFLAGS) -fprofile-use -fprofile-correction"
+ $(MAKE) all CFLAGS_NODIST="$(CFLAGS) $(PGO_PROF_USE_FLAG)"
# Compile and run with gcov
.PHONY=coverage coverage-lcov coverage-report
@@ -1568,9 +1583,11 @@
-rm -f pybuilddir.txt
-rm -f Lib/lib2to3/*Grammar*.pickle
-rm -f Programs/_testembed Programs/_freeze_importlib
+ -rm -rf build
profile-removal:
find . -name '*.gc??' -exec rm -f {} ';'
+ find . -name '*.profclang?' -exec rm -f {} ';'
rm -f $(COVERAGE_INFO)
rm -rf $(COVERAGE_REPORT)
diff --git a/Misc/ACKS b/Misc/ACKS
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -1080,6 +1080,7 @@
Harri Pasanen
Gaël Pasgrimaud
Ashish Nitin Patil
+Alecsandru Patrascu
Randy Pausch
Samuele Pedroni
Justin Peel
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -201,6 +201,9 @@
Build
-----
+- Issue #24915: Add LLVM support for PGO builds and use the test suite to
+ generate the profile data. Initiial patch by Alecsandru Patrascu of Intel.
+
- Issue #24910: Windows MSIs now have unique display names.
- Issue #24986: It is now possible to build Python on Windows without errors
diff --git a/README b/README
--- a/README
+++ b/README
@@ -46,6 +46,34 @@
(This will fail if you *also* built at the top-level directory.
You should do a "make clean" at the toplevel first.)
+If you need an optimized version of Python, you type "make profile-opt" in the
+top level directory. This will rebuild the interpreter executable using Profile
+Guided Optimization (PGO). For more details, see the section bellow.
+
+
+Profile Guided Optimization
+---------------------------
+
+PGO takes advantage of recent versions of the GCC or Clang compilers.
+If ran, the "profile-opt" rule will do several steps.
+
+First, the entire Python directory is cleaned of temporary files that
+may resulted in a previous compilation.
+
+Then, an instrumented version of the interpreter is built, using suitable
+compiler flags for each flavour. Note that this is just an intermediary
+step and the binary resulted after this step is not good for real life
+workloads, as it has profiling instructions embedded inside.
+
+After this instrumented version of the interpreter is built, the Makefile
+will automatically run a training workload. This is necessary in order to
+profile the interpreter execution. Note also that any output, both stdout
+and stderr, that may appear at this step is supressed.
+
+Finally, the last step is to rebuild the interpreter, using the information
+collected in the previous one. The end result will be a the Python binary
+that is optimized and suitable for distribution or production installation.
+
What's New
----------
diff --git a/configure b/configure
--- a/configure
+++ b/configure
@@ -667,6 +667,12 @@
CFLAGS_NODIST
BASECFLAGS
OPT
+LLVM_PROF_FOUND
+LLVM_PROF_ERR
+LLVM_PROF_FILE
+LLVM_PROF_MERGER
+PGO_PROF_USE_FLAG
+PGO_PROF_GEN_FLAG
ABIFLAGS
LN
MKDIR_P
@@ -6431,6 +6437,80 @@
fi
+# Enable PGO flags.
+# Extract the first word of "llvm-profdata", so it can be a program name with args.
+set dummy llvm-profdata; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if ${ac_cv_prog_LLVM_PROF_FOUND+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ if test -n "$LLVM_PROF_FOUND"; then
+ ac_cv_prog_LLVM_PROF_FOUND="$LLVM_PROF_FOUND" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+ IFS=$as_save_IFS
+ test -z "$as_dir" && as_dir=.
+ for ac_exec_ext in '' $ac_executable_extensions; do
+ if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+ ac_cv_prog_LLVM_PROF_FOUND="found"
+ $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+ break 2
+ fi
+done
+ done
+IFS=$as_save_IFS
+
+ test -z "$ac_cv_prog_LLVM_PROF_FOUND" && ac_cv_prog_LLVM_PROF_FOUND="not-found"
+fi
+fi
+LLVM_PROF_FOUND=$ac_cv_prog_LLVM_PROF_FOUND
+if test -n "$LLVM_PROF_FOUND"; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $LLVM_PROF_FOUND" >&5
+$as_echo "$LLVM_PROF_FOUND" >&6; }
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+LLVM_PROF_ERR=no
+case $CC in
+ *clang*)
+ # Any changes made here should be reflected in the GCC+Darwin case below
+ PGO_PROF_GEN_FLAG="-fprofile-instr-generate"
+ PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd"
+ LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr"
+ LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\""
+ if test $LLVM_PROF_FOUND = not-found
+ then
+ LLVM_PROF_ERR=yes
+ fi
+ ;;
+ *gcc*)
+ case $ac_sys_system in
+ Darwin*)
+ PGO_PROF_GEN_FLAG="-fprofile-instr-generate"
+ PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd"
+ LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr"
+ LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\""
+ if test $LLVM_PROF_FOUND = not-found
+ then
+ LLVM_PROF_ERR=yes
+ fi
+ ;;
+ *)
+ PGO_PROF_GEN_FLAG="-fprofile-generate"
+ PGO_PROF_USE_FLAG="-fprofile-use -fprofile-correction"
+ LLVM_PROF_MERGER="true"
+ LLVM_PROF_FILE=""
+ ;;
+ esac
+ ;;
+esac
+
# XXX Shouldn't the code above that fiddles with BASECFLAGS and OPT be
# merged with this chunk of code?
diff --git a/configure.ac b/configure.ac
--- a/configure.ac
+++ b/configure.ac
@@ -1218,6 +1218,49 @@
fi],
[AC_MSG_RESULT(no)])
+# Enable PGO flags.
+AC_SUBST(PGO_PROF_GEN_FLAG)
+AC_SUBST(PGO_PROF_USE_FLAG)
+AC_SUBST(LLVM_PROF_MERGER)
+AC_SUBST(LLVM_PROF_FILE)
+AC_SUBST(LLVM_PROF_ERR)
+AC_SUBST(LLVM_PROF_FOUND)
+AC_CHECK_PROG(LLVM_PROF_FOUND, llvm-profdata, found, not-found)
+LLVM_PROF_ERR=no
+case $CC in
+ *clang*)
+ # Any changes made here should be reflected in the GCC+Darwin case below
+ PGO_PROF_GEN_FLAG="-fprofile-instr-generate"
+ PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd"
+ LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr"
+ LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\""
+ if test $LLVM_PROF_FOUND = not-found
+ then
+ LLVM_PROF_ERR=yes
+ fi
+ ;;
+ *gcc*)
+ case $ac_sys_system in
+ Darwin*)
+ PGO_PROF_GEN_FLAG="-fprofile-instr-generate"
+ PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd"
+ LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr"
+ LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\""
+ if test $LLVM_PROF_FOUND = not-found
+ then
+ LLVM_PROF_ERR=yes
+ fi
+ ;;
+ *)
+ PGO_PROF_GEN_FLAG="-fprofile-generate"
+ PGO_PROF_USE_FLAG="-fprofile-use -fprofile-correction"
+ LLVM_PROF_MERGER="true"
+ LLVM_PROF_FILE=""
+ ;;
+ esac
+ ;;
+esac
+
# XXX Shouldn't the code above that fiddles with BASECFLAGS and OPT be
# merged with this chunk of code?
--
Repository URL: https://hg.python.org/cpython
More information about the Python-checkins
mailing list