[Python-checkins] cpython: Issue #13959: Re-implement imp.load_source() in imp.py.
brett.cannon
python-checkins at python.org
Tue Apr 17 04:11:31 CEST 2012
http://hg.python.org/cpython/rev/3b5b4b4bb43c
changeset: 76371:3b5b4b4bb43c
user: Brett Cannon <brett at python.org>
date: Mon Apr 16 22:11:25 2012 -0400
summary:
Issue #13959: Re-implement imp.load_source() in imp.py.
files:
Lib/imp.py | 29 ++-
Python/import.c | 390 ------------------------------------
2 files changed, 28 insertions(+), 391 deletions(-)
diff --git a/Lib/imp.py b/Lib/imp.py
--- a/Lib/imp.py
+++ b/Lib/imp.py
@@ -14,7 +14,7 @@
from _imp import (get_magic, get_tag, get_suffixes, cache_from_source,
source_from_cache)
# Should be re-implemented here (and mostly deprecated)
-from _imp import (find_module, load_compiled, load_source, NullImporter,
+from _imp import (find_module, load_compiled, NullImporter,
SEARCH_ERROR, PY_SOURCE, PY_COMPILED, C_EXTENSION,
PY_RESOURCE, PKG_DIRECTORY, C_BUILTIN, PY_FROZEN,
PY_CODERESOURCE, IMP_HOOK)
@@ -25,6 +25,33 @@
import os
+class _LoadSourceCompatibility(_bootstrap._SourceFileLoader):
+
+ """Compatibility support for implementing load_source()."""
+
+ def __init__(self, fullname, path, file=None):
+ super().__init__(fullname, path)
+ self.file = file
+
+ def get_data(self, path):
+ """Gross hack to contort SourceFileLoader to deal w/ load_source()'s bad
+ API."""
+ if path == self._path:
+ with self.file:
+ # Technically should be returning bytes, but
+ # SourceLoader.get_code() just passed what is returned to
+ # compile() which can handle str. And converting to bytes would
+ # require figuring out the encoding to decode to and
+ # tokenize.detect_encoding() only accepts bytes.
+ return self.file.read()
+ else:
+ return super().get_data(path)
+
+
+def load_source(name, pathname, file=None):
+ return _LoadSourceCompatibility(name, pathname, file).load_module(name)
+
+
def load_package(name, path):
if os.path.isdir(path):
extensions = _bootstrap._suffix_list(PY_SOURCE)
diff --git a/Python/import.c b/Python/import.c
--- a/Python/import.c
+++ b/Python/import.c
@@ -904,26 +904,6 @@
}
-/* Like strrchr(string, '/') but searches for the rightmost of either SEP
- or ALTSEP, if the latter is defined.
-*/
-static Py_UCS4*
-rightmost_sep(Py_UCS4 *s)
-{
- Py_UCS4 *found, c;
- for (found = NULL; (c = *s); s++) {
- if (c == SEP
-#ifdef ALTSEP
- || c == ALTSEP
-#endif
- )
- {
- found = s;
- }
- }
- return found;
-}
-
/* Like rightmost_sep, but operate on unicode objects. */
static Py_ssize_t
rightmost_sep_obj(PyObject* o, Py_ssize_t start, Py_ssize_t end)
@@ -1081,50 +1061,6 @@
return result;
}
-/* Given a pathname for a Python source file, its time of last
- modification, and a pathname for a compiled file, check whether the
- compiled file represents the same version of the source. If so,
- return a FILE pointer for the compiled file, positioned just after
- the header; if not, return NULL.
- Doesn't set an exception. */
-
-static FILE *
-check_compiled_module(PyObject *pathname, struct stat *srcstat, PyObject *cpathname)
-{
- FILE *fp;
- long magic;
- long pyc_mtime;
- long pyc_size;
-
- fp = _Py_fopen(cpathname, "rb");
- if (fp == NULL)
- return NULL;
- magic = PyMarshal_ReadLongFromFile(fp);
- if (magic != pyc_magic) {
- if (Py_VerboseFlag)
- PySys_FormatStderr("# %R has bad magic\n", cpathname);
- fclose(fp);
- return NULL;
- }
- pyc_mtime = PyMarshal_ReadLongFromFile(fp);
- if (pyc_mtime != srcstat->st_mtime) {
- if (Py_VerboseFlag)
- PySys_FormatStderr("# %R has bad mtime\n", cpathname);
- fclose(fp);
- return NULL;
- }
- pyc_size = PyMarshal_ReadLongFromFile(fp);
- if (pyc_size != (srcstat->st_size & 0xFFFFFFFF)) {
- if (Py_VerboseFlag)
- PySys_FormatStderr("# %R has bad size\n", cpathname);
- fclose(fp);
- return NULL;
- }
- if (Py_VerboseFlag)
- PySys_FormatStderr("# %R matches %R\n", cpathname, pathname);
- return fp;
-}
-
/* Read a code object from a file and check it for validity */
@@ -1178,238 +1114,6 @@
return m;
}
-/* Parse a source file and return the corresponding code object */
-
-static PyCodeObject *
-parse_source_module(PyObject *pathname, FILE *fp)
-{
- PyCodeObject *co;
- PyObject *pathbytes;
- mod_ty mod;
- PyCompilerFlags flags;
- PyArena *arena;
-
- pathbytes = PyUnicode_EncodeFSDefault(pathname);
- if (pathbytes == NULL)
- return NULL;
-
- arena = PyArena_New();
- if (arena == NULL) {
- Py_DECREF(pathbytes);
- return NULL;
- }
-
- flags.cf_flags = 0;
- mod = PyParser_ASTFromFile(fp, PyBytes_AS_STRING(pathbytes), NULL,
- Py_file_input, 0, 0, &flags,
- NULL, arena);
- if (mod != NULL)
- co = PyAST_Compile(mod, PyBytes_AS_STRING(pathbytes), NULL, arena);
- else
- co = NULL;
- Py_DECREF(pathbytes);
- PyArena_Free(arena);
- return co;
-}
-
-/* Write a compiled module to a file, placing the time of last
- modification of its source into the header.
- Errors are ignored, if a write error occurs an attempt is made to
- remove the file. */
-
-static void
-write_compiled_module(PyCodeObject *co, PyObject *cpathname,
- struct stat *srcstat)
-{
- Py_UCS4 *cpathname_ucs4;
- FILE *fp;
- time_t mtime = srcstat->st_mtime;
- long size = srcstat->st_size & 0xFFFFFFFF;
- PyObject *cpathname_tmp;
-#ifdef MS_WINDOWS /* since Windows uses different permissions */
- mode_t mode = srcstat->st_mode & ~S_IEXEC;
- wchar_t *wdirname, *wpathname, *wpathname_tmp;
-#else
- mode_t dirmode = (srcstat->st_mode |
- S_IXUSR | S_IXGRP | S_IXOTH |
- S_IWUSR | S_IWGRP | S_IWOTH);
- PyObject *dirbytes;
- PyObject *cpathbytes, *cpathbytes_tmp;
-#endif
- int fd;
- PyObject *dirname;
- Py_UCS4 *dirsep;
- int res, ok;
-
- /* Ensure that the __pycache__ directory exists. */
- cpathname_ucs4 = PyUnicode_AsUCS4Copy(cpathname);
- if (!cpathname_ucs4)
- return;
- dirsep = rightmost_sep(cpathname_ucs4);
- if (dirsep == NULL) {
- if (Py_VerboseFlag)
- PySys_FormatStderr("# no %s path found %R\n", CACHEDIR, cpathname);
- return;
- }
- dirname = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND,
- cpathname_ucs4,
- dirsep - cpathname_ucs4);
- PyMem_Free(cpathname_ucs4);
- if (dirname == NULL) {
- PyErr_Clear();
- return;
- }
-
-#ifdef MS_WINDOWS
- wdirname = PyUnicode_AsUnicode(dirname);
- if (wdirname == NULL) {
- PyErr_Clear();
- return;
- }
- res = CreateDirectoryW(wdirname, NULL);
- ok = (res != 0);
- if (!ok && GetLastError() == ERROR_ALREADY_EXISTS)
- ok = 1;
-#else
- dirbytes = PyUnicode_EncodeFSDefault(dirname);
- if (dirbytes == NULL) {
- PyErr_Clear();
- return;
- }
- res = mkdir(PyBytes_AS_STRING(dirbytes), dirmode);
- Py_DECREF(dirbytes);
- if (0 <= res)
- ok = 1;
- else
- ok = (errno == EEXIST);
-#endif
- if (!ok) {
- if (Py_VerboseFlag)
- PySys_FormatStderr("# cannot create cache dir %R\n", dirname);
- Py_DECREF(dirname);
- return;
- }
- Py_DECREF(dirname);
-
- /* We first write to a tmp file and then take advantage
- of atomic renaming (which *should* be true even under Windows).
- As in importlib, we use id(something) to generate a pseudo-random
- filename. mkstemp() can't be used since it doesn't allow specifying
- the file access permissions.
- */
- cpathname_tmp = PyUnicode_FromFormat("%U.%zd",
- cpathname, (Py_ssize_t) co);
- if (cpathname_tmp == NULL) {
- PyErr_Clear();
- return;
- }
-#ifdef MS_WINDOWS
- wpathname = PyUnicode_AsUnicode(cpathname);
- if (wpathname == NULL) {
- PyErr_Clear();
- return;
- }
- wpathname_tmp = PyUnicode_AsUnicode(cpathname_tmp);
- if (wpathname_tmp == NULL) {
- PyErr_Clear();
- return;
- }
-
- (void)DeleteFileW(wpathname_tmp);
- fd = _wopen(wpathname_tmp,
- O_EXCL | O_CREAT | O_WRONLY | O_BINARY,
- mode);
- if (0 <= fd)
- fp = fdopen(fd, "wb");
- else
- fp = NULL;
-#else
- cpathbytes_tmp = PyUnicode_EncodeFSDefault(cpathname_tmp);
- Py_DECREF(cpathname_tmp);
- if (cpathbytes_tmp == NULL) {
- PyErr_Clear();
- return;
- }
- cpathbytes = PyUnicode_EncodeFSDefault(cpathname);
- if (cpathbytes == NULL) {
- PyErr_Clear();
- return;
- }
- fd = open(PyBytes_AS_STRING(cpathbytes_tmp),
- O_CREAT | O_EXCL | O_WRONLY, 0666);
- if (0 <= fd)
- fp = fdopen(fd, "wb");
- else
- fp = NULL;
-#endif
- if (fp == NULL) {
- if (Py_VerboseFlag)
- PySys_FormatStderr(
- "# can't create %R\n", cpathname);
-#ifdef MS_WINDOWS
- Py_DECREF(cpathname_tmp);
-#else
- Py_DECREF(cpathbytes);
- Py_DECREF(cpathbytes_tmp);
-#endif
- return;
- }
- PyMarshal_WriteLongToFile(pyc_magic, fp, Py_MARSHAL_VERSION);
- /* First write a 0 for mtime and size */
- PyMarshal_WriteLongToFile(0L, fp, Py_MARSHAL_VERSION);
- PyMarshal_WriteLongToFile(0L, fp, Py_MARSHAL_VERSION);
- PyMarshal_WriteObjectToFile((PyObject *)co, fp, Py_MARSHAL_VERSION);
- fflush(fp);
- /* Now write the true mtime and size (as 32-bit fields) */
- fseek(fp, 4L, 0);
- assert(mtime <= 0xFFFFFFFF);
- PyMarshal_WriteLongToFile((long)mtime, fp, Py_MARSHAL_VERSION);
- PyMarshal_WriteLongToFile(size, fp, Py_MARSHAL_VERSION);
- if (fflush(fp) != 0 || ferror(fp)) {
- if (Py_VerboseFlag)
- PySys_FormatStderr("# can't write %R\n", cpathname);
- /* Don't keep partial file */
- fclose(fp);
-#ifdef MS_WINDOWS
- (void)DeleteFileW(wpathname_tmp);
- Py_DECREF(cpathname_tmp);
-#else
- (void) unlink(PyBytes_AS_STRING(cpathbytes_tmp));
- Py_DECREF(cpathbytes);
- Py_DECREF(cpathbytes_tmp);
-#endif
- return;
- }
- fclose(fp);
- /* Do a (hopefully) atomic rename */
-#ifdef MS_WINDOWS
- if (!MoveFileExW(wpathname_tmp, wpathname, MOVEFILE_REPLACE_EXISTING)) {
- if (Py_VerboseFlag)
- PySys_FormatStderr("# can't write %R\n", cpathname);
- /* Don't keep tmp file */
- (void) DeleteFileW(wpathname_tmp);
- Py_DECREF(cpathname_tmp);
- return;
- }
- Py_DECREF(cpathname_tmp);
-#else
- if (rename(PyBytes_AS_STRING(cpathbytes_tmp),
- PyBytes_AS_STRING(cpathbytes))) {
- if (Py_VerboseFlag)
- PySys_FormatStderr("# can't write %R\n", cpathname);
- /* Don't keep tmp file */
- unlink(PyBytes_AS_STRING(cpathbytes_tmp));
- Py_DECREF(cpathbytes);
- Py_DECREF(cpathbytes_tmp);
- return;
- }
- Py_DECREF(cpathbytes);
- Py_DECREF(cpathbytes_tmp);
-#endif
- if (Py_VerboseFlag)
- PySys_FormatStderr("# wrote %R\n", cpathname);
-}
-
static void
update_code_filenames(PyCodeObject *co, PyObject *oldname, PyObject *newname)
{
@@ -1474,76 +1178,6 @@
Py_RETURN_NONE;
}
-/* Load a source module from a given file and return its module
- object WITH INCREMENTED REFERENCE COUNT. If there's a matching
- byte-compiled file, use that instead. */
-
-static PyObject *
-load_source_module(PyObject *name, PyObject *pathname, FILE *fp)
-{
- struct stat st;
- FILE *fpc;
- PyObject *cpathname = NULL, *cpathbytes = NULL;
- PyCodeObject *co;
- PyObject *m = NULL;
-
- if (fstat(fileno(fp), &st) != 0) {
- PyErr_Format(PyExc_RuntimeError,
- "unable to get file status from %R",
- pathname);
- goto error;
- }
- if (sizeof st.st_mtime > 4) {
- /* Python's .pyc timestamp handling presumes that the timestamp fits
- in 4 bytes. Since the code only does an equality comparison,
- ordering is not important and we can safely ignore the higher bits
- (collisions are extremely unlikely).
- */
- st.st_mtime &= 0xFFFFFFFF;
- }
- if (PyUnicode_READY(pathname) < 0)
- return NULL;
- cpathname = make_compiled_pathname(pathname, !Py_OptimizeFlag);
-
- if (cpathname != NULL)
- fpc = check_compiled_module(pathname, &st, cpathname);
- else
- fpc = NULL;
-
- if (fpc) {
- co = read_compiled_module(cpathname, fpc);
- fclose(fpc);
- if (co == NULL)
- goto error;
- update_compiled_module(co, pathname);
- if (Py_VerboseFlag)
- PySys_FormatStderr("import %U # precompiled from %R\n",
- name, cpathname);
- m = PyImport_ExecCodeModuleObject(name, (PyObject *)co,
- cpathname, cpathname);
- }
- else {
- co = parse_source_module(pathname, fp);
- if (co == NULL)
- goto error;
- if (Py_VerboseFlag)
- PySys_FormatStderr("import %U # from %R\n",
- name, pathname);
- if (cpathname != NULL) {
- PyObject *ro = PySys_GetObject("dont_write_bytecode");
- if (ro == NULL || !PyObject_IsTrue(ro))
- write_compiled_module(co, cpathname, &st);
- }
- m = PyImport_ExecCodeModuleObject(name, (PyObject *)co,
- pathname, cpathname);
- }
- Py_DECREF(co);
-
-error:
- Py_XDECREF(cpathbytes);
- Py_XDECREF(cpathname);
- return m;
-}
/* Get source file -> unicode or None
* Returns the path to the py file if available, else the given path
@@ -3427,29 +3061,6 @@
#endif /* HAVE_DYNAMIC_LOADING */
static PyObject *
-imp_load_source(PyObject *self, PyObject *args)
-{
- PyObject *name, *pathname;
- PyObject *fob = NULL;
- PyObject *m;
- FILE *fp;
- if (!PyArg_ParseTuple(args, "UO&|O:load_source",
- &name,
- PyUnicode_FSDecoder, &pathname,
- &fob))
- return NULL;
- fp = get_file(pathname, fob, "r");
- if (fp == NULL) {
- Py_DECREF(pathname);
- return NULL;
- }
- m = load_source_module(name, pathname, fp);
- Py_DECREF(pathname);
- fclose(fp);
- return m;
-}
-
-static PyObject *
imp_reload(PyObject *self, PyObject *v)
{
return PyImport_ReloadModule(v);
@@ -3600,7 +3211,6 @@
#ifdef HAVE_DYNAMIC_LOADING
{"load_dynamic", imp_load_dynamic, METH_VARARGS},
#endif
- {"load_source", imp_load_source, METH_VARARGS},
{"_fix_co_filename", imp_fix_co_filename, METH_VARARGS},
{NULL, NULL} /* sentinel */
};
--
Repository URL: http://hg.python.org/cpython
More information about the Python-checkins
mailing list