[Pytest-commit] commit/pytest: gutworth: improvements to rewrite cache invalidation
commits-noreply at bitbucket.org
commits-noreply at bitbucket.org
Mon Sep 1 22:51:33 CEST 2014
1 new commit in pytest:
https://bitbucket.org/hpk42/pytest/commits/a433de9ca4f0/
Changeset: a433de9ca4f0
User: gutworth
Date: 2014-09-01 22:51:27
Summary: improvements to rewrite cache invalidation
- stat the source path before it is read.
- Validate the source size in addition to mtime.
Affected #: 3 files
diff -r aed87f841a762133588d09f17f2ceb050d726569 -r a433de9ca4f04e4f4f433ef9e80a5d93790c2f4f CHANGELOG
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,6 +1,8 @@
NEXT
-----------
+- Improve assertion rewriting cache invalidation precision.
+
- fixed issue561: adapt autouse fixture example for python3.
- fixed issue453: assertion rewriting issue with __repr__ containing
diff -r aed87f841a762133588d09f17f2ceb050d726569 -r a433de9ca4f04e4f4f433ef9e80a5d93790c2f4f _pytest/assertion/rewrite.py
--- a/_pytest/assertion/rewrite.py
+++ b/_pytest/assertion/rewrite.py
@@ -134,12 +134,12 @@
co = _read_pyc(fn_pypath, pyc, state.trace)
if co is None:
state.trace("rewriting %r" % (fn,))
- co = _rewrite_test(state, fn_pypath)
+ source_stat, co = _rewrite_test(state, fn_pypath)
if co is None:
# Probably a SyntaxError in the test.
return None
if write:
- _make_rewritten_pyc(state, fn_pypath, pyc, co)
+ _make_rewritten_pyc(state, source_stat, pyc, co)
else:
state.trace("found cached rewritten pyc for %r" % (fn,))
self.modules[name] = co, pyc
@@ -192,13 +192,12 @@
pkg_resources.register_loader_type(cls, pkg_resources.DefaultProvider)
-def _write_pyc(state, co, source_path, pyc):
+def _write_pyc(state, co, source_stat, pyc):
# Technically, we don't have to have the same pyc format as
# (C)Python, since these "pycs" should never be seen by builtin
# import. However, there's little reason deviate, and I hope
# sometime to be able to use imp.load_compiled to load them. (See
# the comment in load_module above.)
- mtime = int(source_path.mtime())
try:
fp = open(pyc, "wb")
except IOError:
@@ -210,7 +209,9 @@
return False
try:
fp.write(imp.get_magic())
- fp.write(struct.pack("<l", mtime))
+ mtime = int(source_stat.mtime)
+ size = source_stat.size & 0xFFFFFFFF
+ fp.write(struct.pack("<ll", mtime, size))
marshal.dump(co, fp)
finally:
fp.close()
@@ -225,9 +226,10 @@
def _rewrite_test(state, fn):
"""Try to read and rewrite *fn* and return the code object."""
try:
+ stat = fn.stat()
source = fn.read("rb")
except EnvironmentError:
- return None
+ return None, None
if ASCII_IS_DEFAULT_ENCODING:
# ASCII is the default encoding in Python 2. Without a coding
# declaration, Python 2 will complain about any bytes in the file
@@ -246,14 +248,15 @@
cookie_re.match(source[0:end1]) is None and
cookie_re.match(source[end1 + 1:end2]) is None):
if hasattr(state, "_indecode"):
- return None # encodings imported us again, we don't rewrite
+ # encodings imported us again, so don't rewrite.
+ return None, None
state._indecode = True
try:
try:
source.decode("ascii")
except UnicodeDecodeError:
# Let it fail in real import.
- return None
+ return None, None
finally:
del state._indecode
# On Python versions which are not 2.7 and less than or equal to 3.1, the
@@ -265,7 +268,7 @@
except SyntaxError:
# Let this pop up again in the real import.
state.trace("failed to parse: %r" % (fn,))
- return None
+ return None, None
rewrite_asserts(tree)
try:
co = compile(tree, fn.strpath, "exec")
@@ -273,20 +276,20 @@
# It's possible that this error is from some bug in the
# assertion rewriting, but I don't know of a fast way to tell.
state.trace("failed to compile: %r" % (fn,))
- return None
- return co
+ return None, None
+ return stat, co
-def _make_rewritten_pyc(state, fn, pyc, co):
+def _make_rewritten_pyc(state, source_stat, pyc, co):
"""Try to dump rewritten code to *pyc*."""
if sys.platform.startswith("win"):
# Windows grants exclusive access to open files and doesn't have atomic
# rename, so just write into the final file.
- _write_pyc(state, co, fn, pyc)
+ _write_pyc(state, co, source_stat, pyc)
else:
# When not on windows, assume rename is atomic. Dump the code object
# into a file specific to this process and atomically replace it.
proc_pyc = pyc + "." + str(os.getpid())
- if _write_pyc(state, co, fn, proc_pyc):
+ if _write_pyc(state, co, source_stat, proc_pyc):
os.rename(proc_pyc, pyc)
def _read_pyc(source, pyc, trace=lambda x: None):
@@ -301,13 +304,14 @@
with fp:
try:
mtime = int(source.mtime())
- data = fp.read(8)
+ size = source.size()
+ data = fp.read(12)
except EnvironmentError as e:
trace('_read_pyc(%s): EnvironmentError %s' % (source, e))
return None
# Check for invalid or out of date pyc file.
- if (len(data) != 8 or data[:4] != imp.get_magic() or
- struct.unpack("<l", data[4:])[0] != mtime):
+ if (len(data) != 12 or data[:4] != imp.get_magic() or
+ struct.unpack("<ll", data[4:]) != (mtime, size)):
trace('_read_pyc(%s): invalid or out of date pyc' % source)
return None
try:
@@ -318,6 +322,7 @@
if not isinstance(co, types.CodeType):
trace('_read_pyc(%s): not a code object' % source)
return None
+ open("/tmp/goop", "wb").write(b"hi")
return co
diff -r aed87f841a762133588d09f17f2ceb050d726569 -r a433de9ca4f04e4f4f433ef9e80a5d93790c2f4f testing/test_assertrewrite.py
--- a/testing/test_assertrewrite.py
+++ b/testing/test_assertrewrite.py
@@ -511,13 +511,13 @@
state = AssertionState(config, "rewrite")
source_path = tmpdir.ensure("source.py")
pycpath = tmpdir.join("pyc").strpath
- assert _write_pyc(state, [1], source_path, pycpath)
+ assert _write_pyc(state, [1], source_path.stat(), pycpath)
def open(*args):
e = IOError()
e.errno = 10
raise e
monkeypatch.setattr(b, "open", open)
- assert not _write_pyc(state, [1], source_path, pycpath)
+ assert not _write_pyc(state, [1], source_path.stat(), pycpath)
def test_resources_provider_for_loader(self, testdir):
"""
Repository URL: https://bitbucket.org/hpk42/pytest/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
More information about the pytest-commit
mailing list