[Python-checkins] cpython: Issue #14928: Fix importlib bootstrap issues by using a custom executable
antoine.pitrou
python-checkins at python.org
Tue Jun 19 22:32:58 CEST 2012
http://hg.python.org/cpython/rev/c3616595dada
changeset: 77519:c3616595dada
user: Antoine Pitrou <solipsis at pitrou.net>
date: Tue Jun 19 22:29:35 2012 +0200
summary:
Issue #14928: Fix importlib bootstrap issues by using a custom executable (Modules/_freeze_importlib) to build Python/importlib.h.
files:
.hgignore | 1 +
Include/pythonrun.h | 3 +
Makefile.pre.in | 31 +-
Misc/NEWS | 6 +
Modules/_freeze_importlib.c | 131 ++
Python/freeze_importlib.py | 39 -
Python/importlib.h | 1281 +++++++++++-----------
Python/pythonrun.c | 11 +-
8 files changed, 806 insertions(+), 697 deletions(-)
diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -79,6 +79,7 @@
PCbuild/amd64
BuildLog.htm
__pycache__
+Modules/_freeze_importlib
Modules/_testembed
.coverage
coverage/
diff --git a/Include/pythonrun.h b/Include/pythonrun.h
--- a/Include/pythonrun.h
+++ b/Include/pythonrun.h
@@ -30,6 +30,9 @@
PyAPI_FUNC(void) Py_Initialize(void);
PyAPI_FUNC(void) Py_InitializeEx(int);
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(void) _Py_InitializeEx_Private(int, int);
+#endif
PyAPI_FUNC(void) Py_Finalize(void);
PyAPI_FUNC(int) Py_IsInitialized(void);
PyAPI_FUNC(PyThreadState *) Py_NewInterpreter(void);
diff --git a/Makefile.pre.in b/Makefile.pre.in
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -324,7 +324,6 @@
Python/codecs.o \
Python/dynamic_annotations.o \
Python/errors.o \
- Python/frozen.o \
Python/frozenmain.o \
Python/future.o \
Python/getargs.o \
@@ -410,7 +409,7 @@
##########################################################################
# objects that get linked into the Python library
-LIBRARY_OBJS= \
+LIBRARY_OBJS_OMIT_FROZEN= \
Modules/getbuildinfo.o \
$(PARSER_OBJS) \
$(OBJECT_OBJS) \
@@ -419,6 +418,10 @@
$(SIGNAL_OBJS) \
$(MODOBJS)
+LIBRARY_OBJS= \
+ $(LIBRARY_OBJS_OMIT_FROZEN) \
+ Python/frozen.o
+
#########################################################################
# Rules
@@ -478,7 +481,7 @@
$(AR) $(ARFLAGS) $@ Modules/getbuildinfo.o
$(AR) $(ARFLAGS) $@ $(PARSER_OBJS)
$(AR) $(ARFLAGS) $@ $(OBJECT_OBJS)
- $(AR) $(ARFLAGS) $@ $(PYTHON_OBJS)
+ $(AR) $(ARFLAGS) $@ $(PYTHON_OBJS) Python/frozen.o
$(AR) $(ARFLAGS) $@ $(MODULE_OBJS) $(SIGNAL_OBJS)
$(AR) $(ARFLAGS) $@ $(MODOBJS)
$(RANLIB) $@
@@ -578,18 +581,14 @@
############################################################################
# Importlib
-Python/importlib.h: $(srcdir)/Lib/importlib/_bootstrap.py $(srcdir)/Python/freeze_importlib.py
- @if test -f ./$(BUILDPYTHON); then \
- $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Python/freeze_importlib.py \
- $(srcdir)/Lib/importlib/_bootstrap.py Python/importlib.h; \
- else \
- echo "----------------------------------------------------------"; \
- echo "Python/importlib.h needs to be rebuilt, but no interpreter"; \
- echo "is available to do so. Leaving the previous version in"; \
- echo "place. You may want to run ''make'' a second time after"; \
- echo "this build is complete."; \
- echo "----------------------------------------------------------"; \
- fi
+Modules/_freeze_importlib: Modules/_freeze_importlib.o $(LIBRARY_OBJS_OMIT_FROZEN)
+ $(LINKCC) $(PY_LDFLAGS) -o $@ Modules/_freeze_importlib.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LIBS) $(MODLIBS) $(SYSLIBS) $(LDLAST)
+
+Python/importlib.h: $(srcdir)/Lib/importlib/_bootstrap.py Modules/_freeze_importlib.c
+ $(MAKE) Modules/_freeze_importlib
+ ./Modules/_freeze_importlib \
+ $(srcdir)/Lib/importlib/_bootstrap.py Python/importlib.h
+
############################################################################
# Special rules for object files
@@ -1389,7 +1388,7 @@
find build -name 'fficonfig.py' -exec rm -f {} ';' || true
-rm -f Lib/lib2to3/*Grammar*.pickle
-rm -f $(SYSCONFIGDATA)
- -rm -f Modules/_testembed
+ -rm -f Modules/_testembed Modules/_freeze_importlib
profile-removal:
find . -name '*.gc??' -exec rm -f {} ';'
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -154,6 +154,12 @@
- Issue #14963: Add test cases for exception handling behaviour
in contextlib.ExitStack (Initial patch by Alon Horev)
+Build
+-----
+
+- Issue #14928: Fix importlib bootstrap issues by using a custom executable
+ (Modules/_freeze_importlib) to build Python/importlib.h.
+
What's New in Python 3.3.0 Alpha 4?
===================================
diff --git a/Modules/_freeze_importlib.c b/Modules/_freeze_importlib.c
new file mode 100644
--- /dev/null
+++ b/Modules/_freeze_importlib.c
@@ -0,0 +1,131 @@
+/* This is built as a stand-alone executable by the Makefile, and helps turn
+ Lib/importlib/_bootstrap.py into a frozen module in Python/importlib.h
+*/
+
+#include <Python.h>
+#include <marshal.h>
+
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+
+/* To avoid a circular dependency on frozen.o, we create our own structure
+ of frozen modules instead, left deliberately blank so as to avoid
+ unintentional import of a stale version of _frozen_importlib. */
+
+static struct _frozen _PyImport_FrozenModules[] = {
+ {0, 0, 0} /* sentinel */
+};
+
+struct _frozen *PyImport_FrozenModules = _PyImport_FrozenModules;
+
+
+const char header[] = "/* Auto-generated by Modules/_freeze_importlib.c */";
+
+int
+main(int argc, char *argv[])
+{
+ char *inpath, *outpath;
+ FILE *infile, *outfile = NULL;
+ struct stat st;
+ size_t text_size, data_size, n;
+ char *text, *data;
+ PyObject *code, *marshalled;
+
+ if (argc != 3) {
+ fprintf(stderr, "need to specify input and output paths\n");
+ return 2;
+ }
+ inpath = argv[1];
+ outpath = argv[2];
+ infile = fopen(inpath, "rb");
+ if (infile == NULL) {
+ fprintf(stderr, "cannot open '%s' for reading\n", inpath);
+ return 1;
+ }
+ if (fstat(fileno(infile), &st)) {
+ fclose(infile);
+ fprintf(stderr, "cannot fstat '%s'\n", inpath);
+ return 1;
+ }
+ text_size = st.st_size;
+ text = (char *) malloc(text_size + 1);
+ if (text == NULL) {
+ fclose(infile);
+ fprintf(stderr, "could not allocate %ld bytes\n", (long) text_size);
+ return 1;
+ }
+ n = fread(text, 1, text_size, infile);
+ fclose(infile);
+ infile = NULL;
+ if (n < text_size) {
+ fprintf(stderr, "read too short: got %ld instead of %ld bytes\n",
+ (long) n, (long) text_size);
+ return 1;
+ }
+ text[text_size] = '\0';
+
+ Py_NoUserSiteDirectory++;
+ Py_NoSiteFlag++;
+ Py_IgnoreEnvironmentFlag++;
+
+ Py_SetProgramName(L"./_freeze_importlib");
+ /* Don't install importlib, since it could execute outdated bytecode. */
+ _Py_InitializeEx_Private(1, 0);
+
+ code = Py_CompileStringExFlags(text, "<frozen importlib._bootstrap>",
+ Py_file_input, NULL, 0);
+ if (code == NULL)
+ goto error;
+ marshalled = PyMarshal_WriteObjectToString(code, Py_MARSHAL_VERSION);
+ Py_DECREF(code);
+ if (marshalled == NULL)
+ goto error;
+
+ assert(PyBytes_CheckExact(marshalled));
+ data = PyBytes_AS_STRING(marshalled);
+ data_size = PyBytes_GET_SIZE(marshalled);
+
+ outfile = fopen(outpath, "wb");
+ if (outfile == NULL) {
+ fprintf(stderr, "cannot open '%s' for writing\n", outpath);
+ return 1;
+ }
+ fprintf(outfile, "%s\n", header);
+ fprintf(outfile, "unsigned char _Py_M__importlib[] = {\n");
+ for (n = 0; n < data_size; n += 16) {
+ size_t i, end = Py_MIN(n + 16, data_size);
+ fprintf(outfile, " ");
+ for (i = n; i < end; i++) {
+ fprintf(outfile, "%d,", (int) data[i]);
+ }
+ fprintf(outfile, "\n");
+ }
+ fprintf(outfile, "};\n");
+
+ Py_DECREF(marshalled);
+
+ Py_Finalize();
+ if (infile)
+ fclose(infile);
+ if (outfile) {
+ if (ferror(outfile)) {
+ fprintf(stderr, "error when writing to '%s'\n", outpath);
+ fclose(outfile);
+ return 1;
+ }
+ fclose(outfile);
+ }
+ return 0;
+
+error:
+ PyErr_Print();
+ Py_Finalize();
+ if (infile)
+ fclose(infile);
+ if (outfile)
+ fclose(outfile);
+ return 1;
+}
diff --git a/Python/freeze_importlib.py b/Python/freeze_importlib.py
deleted file mode 100644
--- a/Python/freeze_importlib.py
+++ /dev/null
@@ -1,39 +0,0 @@
-#! /usr/bin/env python
-"""Freeze importlib for use as the implementation of import."""
-import marshal
-
-
-header = """/* Auto-generated by Python/freeze_importlib.py */"""
-
-
-def main(input_path, output_path):
- with open(input_path, 'r', encoding='utf-8') as input_file:
- source = input_file.read()
-
- code = compile(source, '<frozen importlib._bootstrap>', 'exec')
-
- lines = [header]
- lines.append('unsigned char _Py_M__importlib[] = {')
- data = marshal.dumps(code)
- # Code from Tools/freeze/makefreeze.py:writecode()
- for i in range(0, len(data), 16):
- line = [' ']
- for c in data[i:i+16]:
- line.append('%d,' % c)
- lines.append(''.join(line))
- lines.append('};\n')
- with open(output_path, 'w', encoding='utf-8') as output_file:
- output_file.write('\n'.join(lines))
- # Avoid a compiler warning for lack of EOL
- output_file.write('\n')
-
-
-if __name__ == '__main__':
- import sys
-
- args = sys.argv[1:]
- if len(args) != 2:
- print('Need to specify input and output file paths', file=sys.stderr)
- sys.exit(1)
-
- main(*args)
diff --git a/Python/importlib.h b/Python/importlib.h
--- a/Python/importlib.h
+++ b/Python/importlib.h
[stripped]
diff --git a/Python/pythonrun.c b/Python/pythonrun.c
--- a/Python/pythonrun.c
+++ b/Python/pythonrun.c
@@ -242,7 +242,7 @@
void
-Py_InitializeEx(int install_sigs)
+_Py_InitializeEx_Private(int install_sigs, int install_importlib)
{
PyInterpreterState *interp;
PyThreadState *tstate;
@@ -363,6 +363,9 @@
/* Initialize _warnings. */
_PyWarnings_Init();
+ if (!install_importlib)
+ return;
+
import_init(interp, sysmod);
_PyTime_Init();
@@ -393,6 +396,12 @@
}
void
+Py_InitializeEx(int install_sigs)
+{
+ _Py_InitializeEx_Private(install_sigs, 1);
+}
+
+void
Py_Initialize(void)
{
Py_InitializeEx(1);
--
Repository URL: http://hg.python.org/cpython
More information about the Python-checkins
mailing list