[pypy-svn] r56504 - in pypy/dist/pypy: config module/__builtin__ module/__builtin__/test module/_file rlib
arigo at codespeak.net
arigo at codespeak.net
Sat Jul 12 16:55:23 CEST 2008
Author: arigo
Date: Sat Jul 12 16:55:22 2008
New Revision: 56504
Modified:
pypy/dist/pypy/config/pypyoption.py
pypy/dist/pypy/module/__builtin__/importing.py
pypy/dist/pypy/module/__builtin__/test/test_import.py
pypy/dist/pypy/module/_file/interp_stream.py
pypy/dist/pypy/rlib/streamio.py
Log:
Add an option lonepycfiles, false by default, which allows .pyc
files to be imported even if the corresponding .py file does not
exist.
Sanitize some small parts of the import logic and try to make
it robust against I/O errors.
Modified: pypy/dist/pypy/config/pypyoption.py
==============================================================================
--- pypy/dist/pypy/config/pypyoption.py (original)
+++ pypy/dist/pypy/config/pypyoption.py Sat Jul 12 16:55:22 2008
@@ -137,7 +137,11 @@
BoolOption("usepycfiles", "Write and read pyc files when importing",
default=True),
-
+
+ BoolOption("lonepycfiles", "Import pyc files with no matching py file",
+ default=False,
+ requires=[("objspace.usepycfiles", True)]),
+
BoolOption("honor__builtins__",
"Honor the __builtins__ key of a module dictionary",
default=False),
Modified: pypy/dist/pypy/module/__builtin__/importing.py
==============================================================================
--- pypy/dist/pypy/module/__builtin__/importing.py (original)
+++ pypy/dist/pypy/module/__builtin__/importing.py Sat Jul 12 16:55:22 2008
@@ -10,6 +10,7 @@
from pypy.interpreter.baseobjspace import W_Root, ObjSpace
from pypy.interpreter.eval import Code
from pypy.rlib import streamio
+from pypy.rlib.streamio import StreamErrors
from pypy.rlib.rarithmetic import intmask
from pypy.rlib.objectmodel import we_are_translated
@@ -17,55 +18,34 @@
PYFILE = 1
PYCFILE = 2
-def info_modtype(space, filepart):
- """
- calculate whether the .py file exists, the .pyc file exists
- and whether the .pyc file has the correct mtime entry.
- The latter is only true if the .py file exists.
- The .pyc file is only considered existing if it has a valid
- magic number.
+def find_modtype(space, filepart):
+ """Check which kind of module to import for the given filepart,
+ which is a path without extension. Returns PYFILE, PYCFILE or
+ NOFILE.
"""
+ # check the .py file
pyfile = filepart + ".py"
- pyfile_exist = False
if os.path.exists(pyfile):
pyfile_ts = os.stat(pyfile)[stat.ST_MTIME]
- pyfile_exist = True
+ pyfile_exists = True
else:
+ # The .py file does not exist. By default on PyPy, lonepycfiles
+ # is False: if a .py file does not exist, we don't even try to
+ # look for a lone .pyc file.
+ if not space.config.objspace.lonepycfiles:
+ return NOFILE
pyfile_ts = 0
- pyfile_exist = False
-
- pycfile = filepart + ".pyc"
- if space.config.objspace.usepycfiles and os.path.exists(pycfile):
- pyc_state = check_compiled_module(space, pyfile, pyfile_ts, pycfile)
- pycfile_exists = pyc_state >= 0
- pycfile_ts_valid = pyc_state > 0 or (pyc_state == 0 and not pyfile_exist)
- else:
- pycfile_exists = False
- pycfile_ts_valid = False
-
- return pyfile_exist, pycfile_exists, pycfile_ts_valid
+ pyfile_exists = False
-def find_modtype(space, filepart):
- """ This is the way pypy does it. A pyc is only used if the py file exists AND
- the pyc file contains the timestamp of the py. """
- pyfile_exist, pycfile_exists, pycfile_ts_valid = info_modtype(space, filepart)
- if not pyfile_exist:
- return NOFILE
- elif pycfile_ts_valid:
- return PYCFILE
- else:
- return PYFILE
-
-def find_modtype_cpython(space, filepart):
- """ This is the way cpython does it (where the py file doesnt exist but there
- is a valid pyc file. """
- pyfile_exist, pycfile_exists, pycfile_ts_valid = info_modtype(space, filepart)
- if pycfile_ts_valid:
- return PYCFILE
- elif pyfile_exist:
+ # check the .pyc file
+ if space.config.objspace.usepycfiles:
+ pycfile = filepart + ".pyc"
+ if check_compiled_module(space, pycfile, pyfile_ts):
+ return PYCFILE # existing and up-to-date .pyc file
+
+ # no .pyc file, use the .py file if it exists
+ if pyfile_exists:
return PYFILE
- elif pycfile_exists:
- return PYCFILE
else:
return NOFILE
@@ -88,38 +68,40 @@
w = space.wrap
w_mod = w(Module(space, w_modulename))
- e = None
- if modtype == PYFILE:
- filename = filepart + ".py"
- stream = streamio.open_file_as_stream(filename, "rU")
- else:
- assert modtype == PYCFILE
- filename = filepart + ".pyc"
- stream = streamio.open_file_as_stream(filename, "rb")
-
- _prepare_module(space, w_mod, filename, pkgdir)
try:
+ if modtype == PYFILE:
+ filename = filepart + ".py"
+ stream = streamio.open_file_as_stream(filename, "rU")
+ else:
+ assert modtype == PYCFILE
+ filename = filepart + ".pyc"
+ stream = streamio.open_file_as_stream(filename, "rb")
+
try:
- if modtype == PYFILE:
- load_source_module(space, w_modulename, w_mod, filename, stream.readall())
- else:
- magic = _r_long(stream)
- timestamp = _r_long(stream)
- load_compiled_module(space, w_modulename, w_mod, filename,
- magic, timestamp, stream.readall())
+ _prepare_module(space, w_mod, filename, pkgdir)
+ try:
+ if modtype == PYFILE:
+ load_source_module(space, w_modulename, w_mod, filename,
+ stream.readall())
+ else:
+ magic = _r_long(stream)
+ timestamp = _r_long(stream)
+ load_compiled_module(space, w_modulename, w_mod, filename,
+ magic, timestamp, stream.readall())
+
+ except OperationError, e:
+ w_mods = space.sys.get('modules')
+ space.call_method(w_mods,'pop', w_modulename, space.w_None)
+ raise
finally:
stream.close()
-
- except OperationError, e:
- w_mods = space.sys.get('modules')
- space.call_method(w_mods,'pop', w_modulename, space.w_None)
- raise
-
+
+ except StreamErrors:
+ return None
+
w_mod = check_sys_modules(space, w_modulename)
if w_mod is not None and w_parent is not None:
space.setattr(w_parent, w_name, w_mod)
- if e:
- raise e
return w_mod
def try_getattr(space, w_obj, w_name):
@@ -433,22 +415,25 @@
return w_mod
def _get_long(s):
- if len(s) < 4:
- return -1 # good enough for our purposes
a = ord(s[0])
b = ord(s[1])
c = ord(s[2])
d = ord(s[3])
- x = a | (b<<8) | (c<<16) | (d<<24)
- if _r_correction and d & 0x80 and x > 0:
- x -= _r_correction
- return int(x)
-
-# helper, to avoid exposing internals of marshal and the
-# difficulties of using it though applevel.
-_r_correction = intmask(1L<<32) # == 0 on 32-bit machines
+ if d >= 0x80:
+ d -= 0x100
+ return a | (b<<8) | (c<<16) | (d<<24)
+
+def _read_n(stream, n):
+ buf = ''
+ while len(buf) < n:
+ data = stream.read(n - len(buf))
+ if not data:
+ raise streamio.StreamError("end of file")
+ buf += data
+ return buf
+
def _r_long(stream):
- s = stream.read(4) # XXX XXX could return smaller string
+ s = _read_n(stream, 4)
return _get_long(s)
def _w_long(stream, x):
@@ -461,27 +446,25 @@
d = x & 0xff
stream.write(chr(a) + chr(b) + chr(c) + chr(d))
-def check_compiled_module(space, pathname, mtime, cpathname):
+def check_compiled_module(space, pycfilename, expected_mtime=0):
"""
- Given a pathname for a Python source file, its time of last
- modification, and a pathname for a compiled file, check whether the
- compiled file represents the same version of the source. If so,
- return a FILE pointer for the compiled file, positioned just after
- the header; if not, return NULL.
- Doesn't set an exception.
+ Check if a pyc file's magic number and (optionally) mtime match.
"""
- w_marshal = space.getbuiltinmodule('marshal')
- stream = streamio.open_file_as_stream(cpathname, "rb")
- magic = _r_long(stream)
try:
- if magic != get_pyc_magic(space):
- return -1
- pyc_mtime = _r_long(stream)
- if pyc_mtime != mtime:
- return 0
- finally:
- stream.close()
- return 1
+ stream = streamio.open_file_as_stream(pycfilename, "rb")
+ try:
+ magic = _r_long(stream)
+ if magic != get_pyc_magic(space):
+ return False
+ if expected_mtime != 0:
+ pyc_mtime = _r_long(stream)
+ if pyc_mtime != expected_mtime:
+ return False
+ finally:
+ stream.close()
+ except StreamErrors:
+ return False
+ return True
def read_compiled_module(space, cpathname, strbuf):
""" Read a code object from a file and check it for validity """
@@ -540,7 +523,7 @@
#
try:
stream = streamio.open_file_as_stream(cpathname, "wb")
- except OSError:
+ except StreamErrors:
return # cannot create file
try:
try:
@@ -556,7 +539,7 @@
_w_long(stream, mtime)
finally:
stream.close()
- except OSError:
+ except StreamErrors:
try:
os.unlink(cpathname)
except OSError:
Modified: pypy/dist/pypy/module/__builtin__/test/test_import.py
==============================================================================
--- pypy/dist/pypy/module/__builtin__/test/test_import.py (original)
+++ pypy/dist/pypy/module/__builtin__/test/test_import.py Sat Jul 12 16:55:22 2008
@@ -78,6 +78,9 @@
stream.readall())
finally:
stream.close()
+ if space.config.objspace.usepycfiles:
+ # also create a lone .pyc file
+ p.join('lone.pyc').write(p.join('x.pyc').read())
return str(root)
@@ -297,44 +300,38 @@
def test_check_compiled_module(self):
space = self.space
- pathname = "whatever"
mtime = 12345
cpathname = _testfile(importing.get_pyc_magic(space), mtime)
ret = importing.check_compiled_module(space,
- pathname,
- mtime,
- cpathname)
- assert ret == 1
+ cpathname,
+ mtime)
+ assert ret is True
# check for wrong mtime
ret = importing.check_compiled_module(space,
- pathname,
- mtime+1,
- cpathname)
- assert ret == 0
+ cpathname,
+ mtime+1)
+ assert ret is False
os.remove(cpathname)
# check for wrong version
cpathname = _testfile(importing.get_pyc_magic(space)+1, mtime)
ret = importing.check_compiled_module(space,
- pathname,
- mtime,
- cpathname)
- assert ret == -1
+ cpathname,
+ mtime)
+ assert ret is False
# check for empty .pyc file
f = open(cpathname, 'wb')
f.close()
ret = importing.check_compiled_module(space,
- pathname,
- mtime,
- cpathname)
- assert ret == -1
+ cpathname,
+ mtime)
+ assert ret is False
os.remove(cpathname)
def test_read_compiled_module(self):
space = self.space
- pathname = "whatever"
mtime = 12345
co = compile('x = 42', '?', 'exec')
cpathname = _testfile(importing.get_pyc_magic(space), mtime, co)
@@ -355,7 +352,6 @@
def test_load_compiled_module(self):
space = self.space
- pathname = "whatever"
mtime = 12345
co = compile('x = 42', '?', 'exec')
cpathname = _testfile(importing.get_pyc_magic(space), mtime, co)
@@ -522,12 +518,10 @@
mtime)
# check
- pathname = str(udir.join('cpathname.py'))
ret = importing.check_compiled_module(space,
- pathname,
- mtime,
- cpathname)
- assert ret == 1
+ cpathname,
+ mtime)
+ assert ret is True
# read compiled module
stream = streamio.open_file_as_stream(cpathname, "rb")
@@ -642,3 +636,41 @@
sys.path.pop(0)
sys.path.pop(0)
sys.path_hooks.pop()
+
+class AppTestNoPycFile(object):
+ usepycfiles = False
+ lonepycfiles = False
+
+ def setup_class(cls):
+ cls.space = gettestobjspace(**{
+ "objspace.usepycfiles": cls.usepycfiles,
+ "objspace.lonepycfiles": cls.lonepycfiles,
+ })
+ cls.w_usepycfiles = cls.space.wrap(cls.usepycfiles)
+ cls.w_lonepycfiles = cls.space.wrap(cls.lonepycfiles)
+ cls.saved_modules = _setup(cls.space)
+
+ def teardown_class(cls):
+ _teardown(cls.space, cls.saved_modules)
+
+ def test_import_possibly_from_pyc(self):
+ from compiled import x
+ if self.usepycfiles:
+ assert x.__file__.endswith('x.pyc')
+ else:
+ assert x.__file__.endswith('x.py')
+ try:
+ from compiled import lone
+ except ImportError:
+ assert not self.lonepycfiles, "should have found 'lone.pyc'"
+ else:
+ assert self.lonepycfiles, "should not have found 'lone.pyc'"
+ assert lone.__file__.endswith('lone.pyc')
+
+class AppTestNoLonePycFile(AppTestNoPycFile):
+ usepycfiles = True
+ lonepycfiles = False
+
+class AppTestLonePycFile(AppTestNoPycFile):
+ usepycfiles = True
+ lonepycfiles = True
Modified: pypy/dist/pypy/module/_file/interp_stream.py
==============================================================================
--- pypy/dist/pypy/module/_file/interp_stream.py (original)
+++ pypy/dist/pypy/module/_file/interp_stream.py Sat Jul 12 16:55:22 2008
@@ -1,5 +1,6 @@
import py
from pypy.rlib import streamio
+from pypy.rlib.streamio import StreamErrors
from errno import EINTR
from pypy.interpreter.error import OperationError
@@ -10,8 +11,6 @@
import os
-StreamErrors = (OSError, streamio.StreamError)
-
def wrap_streamerror(space, e):
if isinstance(e, streamio.StreamError):
return OperationError(space.w_ValueError,
Modified: pypy/dist/pypy/rlib/streamio.py
==============================================================================
--- pypy/dist/pypy/rlib/streamio.py (original)
+++ pypy/dist/pypy/rlib/streamio.py Sat Jul 12 16:55:22 2008
@@ -154,6 +154,8 @@
def __init__(self, message):
self.message = message
+StreamErrors = (OSError, StreamError) # errors that can generally be raised
+
class Stream(object):
More information about the Pypy-commit
mailing list