[Python-checkins] cpython (2.7): - Issue #13150, #17512: sysconfig no longer parses the Makefile and config.h
matthias.klose
python-checkins at python.org
Mon Apr 8 21:20:27 CEST 2013
http://hg.python.org/cpython/rev/be3b4aa2ad28
changeset: 83198:be3b4aa2ad28
branch: 2.7
user: doko at ubuntu.com
date: Mon Apr 08 21:20:09 2013 +0200
summary:
- Issue #13150, #17512: sysconfig no longer parses the Makefile and config.h
files when imported, instead doing it at build time. This makes importing
sysconfig faster and reduces Python startup time by 20%.
files:
.bzrignore | 1 +
.hgignore | 1 +
Lib/distutils/sysconfig.py | 63 +--------------------
Lib/pprint.py | 5 +-
Lib/site.py | 15 -----
Lib/sysconfig.py | 75 ++++++++++++++++++++++++-
Makefile.pre.in | 13 +++-
Misc/NEWS | 4 +
Modules/getpath.c | 23 ++++++-
9 files changed, 115 insertions(+), 85 deletions(-)
diff --git a/.bzrignore b/.bzrignore
--- a/.bzrignore
+++ b/.bzrignore
@@ -11,6 +11,7 @@
build
Makefile.pre
platform
+pybuilddir.txt
pyconfig.h
libpython*.a
libpython*.so*
diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -35,6 +35,7 @@
Parser/pgen.stamp$
^core
^python-gdb.py
+^pybuilddir.txt
syntax: glob
python.exe-gdb.py
diff --git a/Lib/distutils/sysconfig.py b/Lib/distutils/sysconfig.py
--- a/Lib/distutils/sysconfig.py
+++ b/Lib/distutils/sysconfig.py
@@ -387,66 +387,11 @@
def _init_posix():
"""Initialize the module as appropriate for POSIX systems."""
- g = {}
- # load the installed Makefile:
- try:
- filename = get_makefile_filename()
- parse_makefile(filename, g)
- except IOError, msg:
- my_msg = "invalid Python installation: unable to open %s" % filename
- if hasattr(msg, "strerror"):
- my_msg = my_msg + " (%s)" % msg.strerror
-
- raise DistutilsPlatformError(my_msg)
-
- # load the installed pyconfig.h:
- try:
- filename = get_config_h_filename()
- parse_config_h(file(filename), g)
- except IOError, msg:
- my_msg = "invalid Python installation: unable to open %s" % filename
- if hasattr(msg, "strerror"):
- my_msg = my_msg + " (%s)" % msg.strerror
-
- raise DistutilsPlatformError(my_msg)
-
- # On AIX, there are wrong paths to the linker scripts in the Makefile
- # -- these paths are relative to the Python source, but when installed
- # the scripts are in another directory.
- if python_build:
- g['LDSHARED'] = g['BLDSHARED']
-
- elif get_python_version() < '2.1':
- # The following two branches are for 1.5.2 compatibility.
- if sys.platform == 'aix4': # what about AIX 3.x ?
- # Linker script is in the config directory, not in Modules as the
- # Makefile says.
- python_lib = get_python_lib(standard_lib=1)
- ld_so_aix = os.path.join(python_lib, 'config', 'ld_so_aix')
- python_exp = os.path.join(python_lib, 'config', 'python.exp')
-
- g['LDSHARED'] = "%s %s -bI:%s" % (ld_so_aix, g['CC'], python_exp)
-
- elif sys.platform == 'beos':
- # Linker script is in the config directory. In the Makefile it is
- # relative to the srcdir, which after installation no longer makes
- # sense.
- python_lib = get_python_lib(standard_lib=1)
- linkerscript_path = string.split(g['LDSHARED'])[0]
- linkerscript_name = os.path.basename(linkerscript_path)
- linkerscript = os.path.join(python_lib, 'config',
- linkerscript_name)
-
- # XXX this isn't the right place to do this: adding the Python
- # library to the link, if needed, should be in the "build_ext"
- # command. (It's also needed for non-MS compilers on Windows, and
- # it's taken care of for them by the 'build_ext.get_libraries()'
- # method.)
- g['LDSHARED'] = ("%s -L%s/lib -lpython%s" %
- (linkerscript, PREFIX, get_python_version()))
-
+ # _sysconfigdata is generated at build time, see the sysconfig module
+ from _sysconfigdata import build_time_vars
global _config_vars
- _config_vars = g
+ _config_vars = {}
+ _config_vars.update(build_time_vars)
def _init_nt():
diff --git a/Lib/pprint.py b/Lib/pprint.py
--- a/Lib/pprint.py
+++ b/Lib/pprint.py
@@ -37,7 +37,10 @@
import sys as _sys
import warnings
-from cStringIO import StringIO as _StringIO
+try:
+ from cStringIO import StringIO as _StringIO
+except ImportError:
+ from StringIO import StringIO as _StringIO
__all__ = ["pprint","pformat","isreadable","isrecursive","saferepr",
"PrettyPrinter"]
diff --git a/Lib/site.py b/Lib/site.py
--- a/Lib/site.py
+++ b/Lib/site.py
@@ -114,18 +114,6 @@
sys.path[:] = L
return known_paths
-# XXX This should not be part of site.py, since it is needed even when
-# using the -S option for Python. See http://www.python.org/sf/586680
-def addbuilddir():
- """Append ./build/lib.<platform> in case we're running in the build dir
- (especially for Guido :-)"""
- from sysconfig import get_platform
- s = "build/lib.%s-%.3s" % (get_platform(), sys.version)
- if hasattr(sys, 'gettotalrefcount'):
- s += '-pydebug'
- s = os.path.join(os.path.dirname(sys.path.pop()), s)
- sys.path.append(s)
-
def _init_pathinfo():
"""Return a set containing all existing directory entries from sys.path"""
@@ -537,9 +525,6 @@
abs__file__()
known_paths = removeduppaths()
- if (os.name == "posix" and sys.path and
- os.path.basename(sys.path[-1]) == "Modules"):
- addbuilddir()
if ENABLE_USER_SITE is None:
ENABLE_USER_SITE = check_enableusersite()
known_paths = addusersitepackages(known_paths)
diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py
--- a/Lib/sysconfig.py
+++ b/Lib/sysconfig.py
@@ -278,9 +278,10 @@
return os.path.join(_PROJECT_BASE, "Makefile")
return os.path.join(get_path('platstdlib'), "config", "Makefile")
-
-def _init_posix(vars):
- """Initialize the module as appropriate for POSIX systems."""
+def _generate_posix_vars():
+ """Generate the Python module containing build-time variables."""
+ import pprint
+ vars = {}
# load the installed Makefile:
makefile = _get_makefile_filename()
try:
@@ -308,6 +309,49 @@
if _PYTHON_BUILD:
vars['LDSHARED'] = vars['BLDSHARED']
+ # There's a chicken-and-egg situation on OS X with regards to the
+ # _sysconfigdata module after the changes introduced by #15298:
+ # get_config_vars() is called by get_platform() as part of the
+ # `make pybuilddir.txt` target -- which is a precursor to the
+ # _sysconfigdata.py module being constructed. Unfortunately,
+ # get_config_vars() eventually calls _init_posix(), which attempts
+ # to import _sysconfigdata, which we won't have built yet. In order
+ # for _init_posix() to work, if we're on Darwin, just mock up the
+ # _sysconfigdata module manually and populate it with the build vars.
+ # This is more than sufficient for ensuring the subsequent call to
+ # get_platform() succeeds.
+ name = '_sysconfigdata'
+ if 'darwin' in sys.platform:
+ import imp
+ module = imp.new_module(name)
+ module.build_time_vars = vars
+ sys.modules[name] = module
+
+ pybuilddir = 'build/lib.%s-%s' % (get_platform(), sys.version[:3])
+ if hasattr(sys, "gettotalrefcount"):
+ pybuilddir += '-pydebug'
+ try:
+ os.makedirs(pybuilddir)
+ except OSError:
+ pass
+ destfile = os.path.join(pybuilddir, name + '.py')
+
+ with open(destfile, 'wb') as f:
+ f.write('# system configuration generated and used by'
+ ' the sysconfig module\n')
+ f.write('build_time_vars = ')
+ pprint.pprint(vars, stream=f)
+
+ # Create file used for sys.path fixup -- see Modules/getpath.c
+ with open('pybuilddir.txt', 'w') as f:
+ f.write(pybuilddir)
+
+def _init_posix(vars):
+ """Initialize the module as appropriate for POSIX systems."""
+ # _sysconfigdata is generated at build time, see _generate_posix_vars()
+ from _sysconfigdata import build_time_vars
+ vars.update(build_time_vars)
+
def _init_non_posix(vars):
"""Initialize the module as appropriate for NT"""
# set basic install directories
@@ -565,3 +609,28 @@
def get_python_version():
return _PY_VERSION_SHORT
+
+
+def _print_dict(title, data):
+ for index, (key, value) in enumerate(sorted(data.items())):
+ if index == 0:
+ print '%s: ' % (title)
+ print '\t%s = "%s"' % (key, value)
+
+
+def _main():
+ """Display all information sysconfig detains."""
+ if '--generate-posix-vars' in sys.argv:
+ _generate_posix_vars()
+ return
+ print 'Platform: "%s"' % get_platform()
+ print 'Python version: "%s"' % get_python_version()
+ print 'Current installation scheme: "%s"' % _get_default_scheme()
+ print
+ _print_dict('Paths', get_paths())
+ print
+ _print_dict('Variables', get_config_vars())
+
+
+if __name__ == '__main__':
+ _main()
diff --git a/Makefile.pre.in b/Makefile.pre.in
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -437,15 +437,20 @@
Modules/python.o \
$(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) $(LDLAST)
-platform: $(BUILDPYTHON)
+platform: $(BUILDPYTHON) pybuilddir.txt
$(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print get_platform()+"-"+sys.version[0:3]' >platform
+# Create build directory and generate the sysconfig build-time data there.
+# pybuilddir.txt contains the name of the build dir and is used for
+# sys.path fixup -- see Modules/getpath.c.
+pybuilddir.txt: $(BUILDPYTHON)
+ $(RUNSHARED) $(PYTHON_FOR_BUILD) -S -m sysconfig --generate-posix-vars
# Build the shared modules
# Under GNU make, MAKEFLAGS are sorted and normalized; the 's' for
# -s, --silent or --quiet is always the first char.
# Under BSD make, MAKEFLAGS might be " -s -v x=y".
-sharedmods: $(BUILDPYTHON)
+sharedmods: $(BUILDPYTHON) pybuilddir.txt
@case "$$MAKEFLAGS" in \
*\ -s*|s*) quiet="-q";; \
*) quiet="";; \
@@ -955,7 +960,7 @@
else true; \
fi; \
done
- @for i in $(srcdir)/Lib/*.py $(srcdir)/Lib/*.doc $(srcdir)/Lib/*.egg-info ; \
+ @for i in $(srcdir)/Lib/*.py `cat pybuilddir.txt`/_sysconfigdata.py $(srcdir)/Lib/*.doc $(srcdir)/Lib/*.egg-info ; \
do \
if test -x $$i; then \
$(INSTALL_SCRIPT) $$i $(DESTDIR)$(LIBDEST); \
@@ -1133,6 +1138,7 @@
--install-scripts=$(BINDIR) \
--install-platlib=$(DESTSHARED) \
--root=$(DESTDIR)/
+ -rm $(DESTDIR)$(DESTSHARED)/_sysconfigdata.py*
# Here are a couple of targets for MacOSX again, to install a full
# framework-based Python. frameworkinstall installs everything, the
@@ -1288,6 +1294,7 @@
Modules/Setup Modules/Setup.local Modules/Setup.config \
Modules/ld_so_aix Modules/python.exp Misc/python.pc
-rm -f python*-gdb.py
+ -rm -f pybuilddir.txt
find $(srcdir) '(' -name '*.fdc' -o -name '*~' \
-o -name '[@,#]*' -o -name '*.old' \
-o -name '*.orig' -o -name '*.rej' \
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -14,6 +14,10 @@
Library
-------
+- Issue #13150, #17512: sysconfig no longer parses the Makefile and config.h
+ files when imported, instead doing it at build time. This makes importing
+ sysconfig faster and reduces Python startup time by 20%.
+
- Issue #13163: Rename operands in smtplib.SMTP._get_socket to correct names;
fixes otherwise misleading output in tracebacks and when when debug is on.
diff --git a/Modules/getpath.c b/Modules/getpath.c
--- a/Modules/getpath.c
+++ b/Modules/getpath.c
@@ -335,12 +335,27 @@
return 1;
}
- /* Check to see if argv[0] is in the build directory */
+ /* Check to see if argv[0] is in the build directory. "pybuilddir.txt"
+ is written by setup.py and contains the relative path to the location
+ of shared library modules. */
strcpy(exec_prefix, argv0_path);
- joinpath(exec_prefix, "Modules/Setup");
+ joinpath(exec_prefix, "pybuilddir.txt");
if (isfile(exec_prefix)) {
- reduce(exec_prefix);
- return -1;
+ FILE *f = fopen(exec_prefix, "r");
+ if (f == NULL)
+ errno = 0;
+ else {
+ char rel_builddir_path[MAXPATHLEN+1];
+ size_t n;
+ n = fread(rel_builddir_path, 1, MAXPATHLEN, f);
+ rel_builddir_path[n] = '\0';
+ fclose(f);
+ if (n >= 0) {
+ strcpy(exec_prefix, argv0_path);
+ joinpath(exec_prefix, rel_builddir_path);
+ return -1;
+ }
+ }
}
/* Search from argv0_path, until root is found */
--
Repository URL: http://hg.python.org/cpython
More information about the Python-checkins
mailing list