[pypy-commit] pypy py3.3: merge upstream
pjenvey
noreply at buildbot.pypy.org
Mon Oct 27 06:19:39 CET 2014
Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3.3
Changeset: r74256:376dcdcf68d7
Date: 2014-10-26 22:19 -0700
http://bitbucket.org/pypy/pypy/changeset/376dcdcf68d7/
Log: merge upstream
diff --git a/pypy/module/_lzma/__init__.py b/pypy/module/_lzma/__init__.py
--- a/pypy/module/_lzma/__init__.py
+++ b/pypy/module/_lzma/__init__.py
@@ -11,11 +11,17 @@
'LZMAError': 'interp_lzma.W_LZMAError',
'_encode_filter_properties': 'interp_lzma.encode_filter_properties',
'_decode_filter_properties': 'interp_lzma.decode_filter_properties',
- 'FORMAT_AUTO': 'space.wrap(interp_lzma.FORMAT_AUTO)',
- 'FORMAT_XZ': 'space.wrap(interp_lzma.FORMAT_XZ)',
- 'FORMAT_ALONE': 'space.wrap(interp_lzma.FORMAT_ALONE)',
- 'FORMAT_RAW': 'space.wrap(interp_lzma.FORMAT_RAW)',
}
+ for name in 'AUTO XZ ALONE RAW'.split():
+ interpleveldefs['FORMAT_%s' % name] = (
+ 'space.wrap(interp_lzma.FORMAT_%s)' % name)
+ for name in 'DEFAULT EXTREME'.split():
+ interpleveldefs['PRESET_%s' % name] = (
+ 'space.wrap(interp_lzma.LZMA_PRESET_%s)' % name)
+ for name in 'LZMA1 LZMA2 DELTA X86 IA64 ARM ARMTHUMB SPARC POWERPC'.split():
+ interpleveldefs['FILTER_%s' % name] = (
+ 'space.wrap(interp_lzma.LZMA_FILTER_%s)' % name)
+
appleveldefs = {
}
diff --git a/pypy/module/_lzma/interp_lzma.py b/pypy/module/_lzma/interp_lzma.py
--- a/pypy/module/_lzma/interp_lzma.py
+++ b/pypy/module/_lzma/interp_lzma.py
@@ -54,6 +54,10 @@
LZMA_PRESET_DEFAULT
LZMA_CHECK_ID_MAX
LZMA_TELL_ANY_CHECK LZMA_TELL_NO_CHECK
+ LZMA_FILTER_LZMA1 LZMA_FILTER_LZMA2 LZMA_FILTER_DELTA
+ LZMA_FILTER_IA64 LZMA_FILTER_X86 LZMA_FILTER_ARM LZMA_FILTER_ARMTHUMB
+ LZMA_FILTER_SPARC LZMA_FILTER_POWERPC
+ LZMA_PRESET_DEFAULT LZMA_PRESET_EXTREME
'''.split()
for name in constant_names:
setattr(CConfig, name, platform.ConstantInteger(name))
@@ -187,7 +191,7 @@
raise_error(space, "Invalid compression preset: %d", preset)
lzret = lzma_alone_encoder(self.lzs, options)
else:
- raise NotImplementedError
+ raise oefmt(space.w_NotImplementedError, "Filter specs")
_catch_lzma_error(space, lzret)
@staticmethod
@@ -209,7 +213,7 @@
if format == FORMAT_ALONE:
self._init_alone(space, preset, w_filters)
else:
- raise NotImplementedError
+ raise oefmt(space.w_NotImplementedError, "Format %d", format)
return w_self
@@ -294,7 +298,7 @@
lzret = lzma_auto_decoder(self.lzs, memlimit, decoder_flags)
_catch_lzma_error(space, lzret)
else:
- raise NotImplementedError
+ raise oefmt(space.w_NotImplementedError, "Format %d", format)
return w_self
diff --git a/pypy/module/marshal/interp_marshal.py b/pypy/module/marshal/interp_marshal.py
--- a/pypy/module/marshal/interp_marshal.py
+++ b/pypy/module/marshal/interp_marshal.py
@@ -1,4 +1,4 @@
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, oefmt
from pypy.interpreter.gateway import WrappedDefault, unwrap_spec
from rpython.rlib.rarithmetic import intmask
from rpython.rlib import rstackovf
@@ -102,8 +102,13 @@
space = self.space
w_ret = space.call_function(self.func, space.wrap(n))
ret = space.str_w(w_ret)
- if len(ret) != n:
+ if len(ret) < n:
self.raise_eof()
+ if len(ret) > n:
+ raise oefmt(space.w_ValueError,
+ "read() returned too much data: "
+ "%d bytes requested, %d returned",
+ n, len(ret))
return ret
diff --git a/pypy/module/marshal/test/test_marshal.py b/pypy/module/marshal/test/test_marshal.py
--- a/pypy/module/marshal/test/test_marshal.py
+++ b/pypy/module/marshal/test/test_marshal.py
@@ -205,6 +205,18 @@
raises(EOFError, marshal.loads, b'<test>')
raises((MemoryError, ValueError), marshal.loads, b'(test)')
+ def test_bad_reader(self):
+ import marshal, io
+ class BadReader(io.BytesIO):
+ def read(self, n=-1):
+ b = super().read(n)
+ if n is not None and n > 4:
+ b += b' ' * 10**6
+ return b
+ for value in (1.0, 1j, b'0123456789', '0123456789'):
+ raises(ValueError, marshal.load,
+ BadReader(marshal.dumps(value)))
+
class AppTestSmallLong(AppTestMarshal):
spaceconfig = AppTestMarshal.spaceconfig.copy()
diff --git a/pypy/module/zlib/interp_zlib.py b/pypy/module/zlib/interp_zlib.py
--- a/pypy/module/zlib/interp_zlib.py
+++ b/pypy/module/zlib/interp_zlib.py
@@ -117,11 +117,14 @@
method=rzlib.Z_DEFLATED, # \
wbits=rzlib.MAX_WBITS, # \ undocumented
memLevel=rzlib.DEF_MEM_LEVEL, # / parameters
- strategy=rzlib.Z_DEFAULT_STRATEGY): # /
+ strategy=rzlib.Z_DEFAULT_STRATEGY, # /
+ zdict=None):
ZLibObject.__init__(self, space)
try:
self.stream = rzlib.deflateInit(level, method, wbits,
memLevel, strategy)
+ if zdict is not None:
+ rzlib.deflateSetDictionary(self.stream, zdict)
except rzlib.RZlibError, e:
raise zlib_error(space, e.msg)
except ValueError:
@@ -192,14 +195,19 @@
method=rzlib.Z_DEFLATED, # \
wbits=rzlib.MAX_WBITS, # \ undocumented
memLevel=rzlib.DEF_MEM_LEVEL, # / parameters
- strategy=rzlib.Z_DEFAULT_STRATEGY): # /
+ strategy=rzlib.Z_DEFAULT_STRATEGY, # /
+ w_zdict=None):
"""
Create a new z_stream and call its initializer.
"""
+ if space.is_none(w_zdict):
+ zdict = None
+ else:
+ zdict = space.bufferstr_w(w_zdict)
stream = space.allocate_instance(Compress, w_subtype)
stream = space.interp_w(Compress, stream)
Compress.__init__(stream, space, level,
- method, wbits, memLevel, strategy)
+ method, wbits, memLevel, strategy, zdict)
return space.wrap(stream)
@@ -219,7 +227,7 @@
Wrapper around zlib's z_stream structure which provides convenient
decompression functionality.
"""
- def __init__(self, space, wbits=rzlib.MAX_WBITS):
+ def __init__(self, space, wbits=rzlib.MAX_WBITS, zdict=None):
"""
Initialize a new decompression object.
@@ -239,7 +247,8 @@
except ValueError:
raise OperationError(space.w_ValueError,
space.wrap("Invalid initialization option"))
-
+ self.zdict = zdict
+
def __del__(self):
"""Automatically free the resources used by the stream."""
if self.stream:
@@ -274,7 +283,9 @@
try:
self.lock()
try:
- result = rzlib.decompress(self.stream, data, max_length=max_length)
+ result = rzlib.decompress(self.stream, data,
+ max_length=max_length,
+ zdict=self.zdict)
finally:
self.unlock()
except rzlib.RZlibError, e:
@@ -300,7 +311,8 @@
try:
self.lock()
try:
- result = rzlib.decompress(self.stream, data, rzlib.Z_FINISH)
+ result = rzlib.decompress(self.stream, data, rzlib.Z_FINISH,
+ zdict=self.zdict)
finally:
self.unlock()
except rzlib.RZlibError:
@@ -312,13 +324,17 @@
@unwrap_spec(wbits=int)
-def Decompress___new__(space, w_subtype, wbits=rzlib.MAX_WBITS):
+def Decompress___new__(space, w_subtype, wbits=rzlib.MAX_WBITS, w_zdict=None):
"""
Create a new Decompress and call its initializer.
"""
+ if space.is_none(w_zdict):
+ zdict = None
+ else:
+ zdict = space.bufferstr_w(w_zdict)
stream = space.allocate_instance(Decompress, w_subtype)
stream = space.interp_w(Decompress, stream)
- Decompress.__init__(stream, space, wbits)
+ Decompress.__init__(stream, space, wbits, zdict)
return space.wrap(stream)
diff --git a/pypy/module/zlib/test/test_zlib.py b/pypy/module/zlib/test/test_zlib.py
--- a/pypy/module/zlib/test/test_zlib.py
+++ b/pypy/module/zlib/test/test_zlib.py
@@ -2,10 +2,13 @@
Tests for the zlib module.
"""
+import py
+import pypy
+
try:
import zlib
except ImportError:
- import py; py.test.skip("no zlib module on this host Python")
+ py.test.skip("no zlib module on this host Python")
class AppTestZlib(object):
@@ -22,6 +25,9 @@
expanded = b'some bytes which will be compressed'
cls.w_expanded = cls.space.wrapbytes(expanded)
cls.w_compressed = cls.space.wrapbytes(zlib.compress(expanded))
+ cls.w_LICENSE = cls.space.wrapbytes(
+ py.path.local(pypy.__file__).dirpath().dirpath()
+ .join('LICENSE').read())
def test_error(self):
"""
@@ -275,3 +281,31 @@
assert dco.flush(1) == input1[1:]
assert dco.unused_data == b''
assert dco.unconsumed_tail == b''
+
+ def test_dictionary(self):
+ l = self.LICENSE
+ # Build a simulated dictionary out of the words in LICENSE.
+ words = l.split()
+ zdict = b''.join(set(words))
+ # Use it to compress LICENSE.
+ co = self.zlib.compressobj(zdict=zdict)
+ cd = co.compress(l) + co.flush()
+ # Verify that it will decompress with the dictionary.
+ dco = self.zlib.decompressobj(zdict=zdict)
+ assert dco.decompress(cd) + dco.flush() == l
+ # Verify that it fails when not given the dictionary.
+ dco = self.zlib.decompressobj()
+ raises(self.zlib.error, dco.decompress, cd)
+
+ def test_dictionary_streaming(self):
+ # This simulates the reuse of a compressor object for compressing
+ # several separate data streams.
+ co = self.zlib.compressobj(zdict=self.LICENSE)
+ do = self.zlib.decompressobj(zdict=self.LICENSE)
+ piece = self.LICENSE[1000:1500]
+ d0 = co.compress(piece) + co.flush(self.zlib.Z_SYNC_FLUSH)
+ d1 = co.compress(piece[100:]) + co.flush(self.zlib.Z_SYNC_FLUSH)
+ d2 = co.compress(piece[:-100]) + co.flush(self.zlib.Z_SYNC_FLUSH)
+ assert do.decompress(d0) == piece
+ do.decompress(d1) == piece[100:]
+ do.decompress(d2) == piece[:-100]
diff --git a/rpython/rlib/rzlib.py b/rpython/rlib/rzlib.py
--- a/rpython/rlib/rzlib.py
+++ b/rpython/rlib/rzlib.py
@@ -322,7 +322,8 @@
return data
-def decompress(stream, data, flush=Z_SYNC_FLUSH, max_length=sys.maxint):
+def decompress(stream, data, flush=Z_SYNC_FLUSH, max_length=sys.maxint,
+ zdict=None):
"""
Feed more data into an inflate stream. Returns a tuple (string,
finished, unused_data_length). The string contains (a part of) the
@@ -348,7 +349,7 @@
should_finish = False
while_doing = "while decompressing data"
data, err, avail_in = _operate(stream, data, flush, max_length, _inflate,
- while_doing)
+ while_doing, zdict=zdict)
if should_finish:
# detect incomplete input
rffi.setintfield(stream, 'c_avail_in', 0)
@@ -359,7 +360,7 @@
return data, finished, avail_in
-def _operate(stream, data, flush, max_length, cfunc, while_doing):
+def _operate(stream, data, flush, max_length, cfunc, while_doing, zdict=None):
"""Common code for compress() and decompress().
"""
# Prepare the input buffer for the stream
@@ -388,6 +389,10 @@
max_length -= bufsize
rffi.setintfield(stream, 'c_avail_out', bufsize)
err = cfunc(stream, flush)
+ if err == Z_NEED_DICT and zdict is not None:
+ inflateSetDictionary(stream, zdict)
+ # repeat the call to inflate
+ err = cfunc(stream, flush)
if err == Z_OK or err == Z_STREAM_END:
# accumulate data into 'result'
avail_out = rffi.cast(lltype.Signed, stream.c_avail_out)
More information about the pypy-commit
mailing list