[pypy-commit] pypy py3.3: zlib: Add support for the "zdict" parameter.
amauryfa
noreply at buildbot.pypy.org
Sun Oct 26 19:32:10 CET 2014
Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.3
Changeset: r74252:c2fcbb0f5a75
Date: 2014-10-26 19:31 +0100
http://bitbucket.org/pypy/pypy/changeset/c2fcbb0f5a75/
Log: zlib: Add support for the "zdict" parameter.
diff --git a/pypy/module/zlib/interp_zlib.py b/pypy/module/zlib/interp_zlib.py
--- a/pypy/module/zlib/interp_zlib.py
+++ b/pypy/module/zlib/interp_zlib.py
@@ -117,11 +117,14 @@
method=rzlib.Z_DEFLATED, # \
wbits=rzlib.MAX_WBITS, # \ undocumented
memLevel=rzlib.DEF_MEM_LEVEL, # / parameters
- strategy=rzlib.Z_DEFAULT_STRATEGY): # /
+ strategy=rzlib.Z_DEFAULT_STRATEGY, # /
+ zdict=None):
ZLibObject.__init__(self, space)
try:
self.stream = rzlib.deflateInit(level, method, wbits,
memLevel, strategy)
+ if zdict is not None:
+ rzlib.deflateSetDictionary(self.stream, zdict)
except rzlib.RZlibError, e:
raise zlib_error(space, e.msg)
except ValueError:
@@ -192,14 +195,19 @@
method=rzlib.Z_DEFLATED, # \
wbits=rzlib.MAX_WBITS, # \ undocumented
memLevel=rzlib.DEF_MEM_LEVEL, # / parameters
- strategy=rzlib.Z_DEFAULT_STRATEGY): # /
+ strategy=rzlib.Z_DEFAULT_STRATEGY, # /
+ w_zdict=None):
"""
Create a new z_stream and call its initializer.
"""
+ if space.is_none(w_zdict):
+ zdict = None
+ else:
+ zdict = space.bufferstr_w(w_zdict)
stream = space.allocate_instance(Compress, w_subtype)
stream = space.interp_w(Compress, stream)
Compress.__init__(stream, space, level,
- method, wbits, memLevel, strategy)
+ method, wbits, memLevel, strategy, zdict)
return space.wrap(stream)
@@ -219,7 +227,7 @@
Wrapper around zlib's z_stream structure which provides convenient
decompression functionality.
"""
- def __init__(self, space, wbits=rzlib.MAX_WBITS):
+ def __init__(self, space, wbits=rzlib.MAX_WBITS, zdict=None):
"""
Initialize a new decompression object.
@@ -239,7 +247,8 @@
except ValueError:
raise OperationError(space.w_ValueError,
space.wrap("Invalid initialization option"))
-
+ self.zdict = zdict
+
def __del__(self):
"""Automatically free the resources used by the stream."""
if self.stream:
@@ -274,7 +283,9 @@
try:
self.lock()
try:
- result = rzlib.decompress(self.stream, data, max_length=max_length)
+ result = rzlib.decompress(self.stream, data,
+ max_length=max_length,
+ zdict=self.zdict)
finally:
self.unlock()
except rzlib.RZlibError, e:
@@ -300,7 +311,8 @@
try:
self.lock()
try:
- result = rzlib.decompress(self.stream, data, rzlib.Z_FINISH)
+ result = rzlib.decompress(self.stream, data, rzlib.Z_FINISH,
+ zdict=self.zdict)
finally:
self.unlock()
except rzlib.RZlibError:
@@ -312,13 +324,17 @@
@unwrap_spec(wbits=int)
-def Decompress___new__(space, w_subtype, wbits=rzlib.MAX_WBITS):
+def Decompress___new__(space, w_subtype, wbits=rzlib.MAX_WBITS, w_zdict=None):
"""
Create a new Decompress and call its initializer.
"""
+ if space.is_none(w_zdict):
+ zdict = None
+ else:
+ zdict = space.bufferstr_w(w_zdict)
stream = space.allocate_instance(Decompress, w_subtype)
stream = space.interp_w(Decompress, stream)
- Decompress.__init__(stream, space, wbits)
+ Decompress.__init__(stream, space, wbits, zdict)
return space.wrap(stream)
diff --git a/pypy/module/zlib/test/test_zlib.py b/pypy/module/zlib/test/test_zlib.py
--- a/pypy/module/zlib/test/test_zlib.py
+++ b/pypy/module/zlib/test/test_zlib.py
@@ -2,10 +2,13 @@
Tests for the zlib module.
"""
+import py
+import pypy
+
try:
import zlib
except ImportError:
- import py; py.test.skip("no zlib module on this host Python")
+ py.test.skip("no zlib module on this host Python")
class AppTestZlib(object):
@@ -22,6 +25,9 @@
expanded = b'some bytes which will be compressed'
cls.w_expanded = cls.space.wrapbytes(expanded)
cls.w_compressed = cls.space.wrapbytes(zlib.compress(expanded))
+ cls.w_LICENSE = cls.space.wrapbytes(
+ py.path.local(pypy.__file__).dirpath().dirpath()
+ .join('LICENSE').read())
def test_error(self):
"""
@@ -275,3 +281,31 @@
assert dco.flush(1) == input1[1:]
assert dco.unused_data == b''
assert dco.unconsumed_tail == b''
+
+ def test_dictionary(self):
+ l = self.LICENSE
+ # Build a simulated dictionary out of the words in LICENSE.
+ words = l.split()
+ zdict = b''.join(set(words))
+ # Use it to compress LICENSE.
+ co = self.zlib.compressobj(zdict=zdict)
+ cd = co.compress(l) + co.flush()
+ # Verify that it will decompress with the dictionary.
+ dco = self.zlib.decompressobj(zdict=zdict)
+ assert dco.decompress(cd) + dco.flush() == l
+ # Verify that it fails when not given the dictionary.
+ dco = self.zlib.decompressobj()
+ raises(self.zlib.error, dco.decompress, cd)
+
+ def test_dictionary_streaming(self):
+ # This simulates the reuse of a compressor object for compressing
+ # several separate data streams.
+ co = self.zlib.compressobj(zdict=self.LICENSE)
+ do = self.zlib.decompressobj(zdict=self.LICENSE)
+ piece = self.LICENSE[1000:1500]
+ d0 = co.compress(piece) + co.flush(self.zlib.Z_SYNC_FLUSH)
+ d1 = co.compress(piece[100:]) + co.flush(self.zlib.Z_SYNC_FLUSH)
+ d2 = co.compress(piece[:-100]) + co.flush(self.zlib.Z_SYNC_FLUSH)
+ assert do.decompress(d0) == piece
+ do.decompress(d1) == piece[100:]
+ do.decompress(d2) == piece[:-100]
diff --git a/rpython/rlib/rzlib.py b/rpython/rlib/rzlib.py
--- a/rpython/rlib/rzlib.py
+++ b/rpython/rlib/rzlib.py
@@ -330,7 +330,8 @@
return data
-def decompress(stream, data, flush=Z_SYNC_FLUSH, max_length=sys.maxint):
+def decompress(stream, data, flush=Z_SYNC_FLUSH, max_length=sys.maxint,
+ zdict=None):
"""
Feed more data into an inflate stream. Returns a tuple (string,
finished, unused_data_length). The string contains (a part of) the
@@ -356,7 +357,7 @@
should_finish = False
while_doing = "while decompressing data"
data, err, avail_in = _operate(stream, data, flush, max_length, _inflate,
- while_doing)
+ while_doing, zdict=zdict)
if should_finish:
# detect incomplete input
rffi.setintfield(stream, 'c_avail_in', 0)
@@ -367,7 +368,7 @@
return data, finished, avail_in
-def _operate(stream, data, flush, max_length, cfunc, while_doing):
+def _operate(stream, data, flush, max_length, cfunc, while_doing, zdict=None):
"""Common code for compress() and decompress().
"""
# Prepare the input buffer for the stream
@@ -394,6 +395,10 @@
max_length -= bufsize
rffi.setintfield(stream, 'c_avail_out', bufsize)
err = cfunc(stream, flush)
+ if err == Z_NEED_DICT and zdict is not None:
+ inflateSetDictionary(stream, zdict)
+ # repeat the call to inflate
+ err = cfunc(stream, flush)
if err == Z_OK or err == Z_STREAM_END:
# accumulate data into 'result'
avail_out = rffi.cast(lltype.Signed, stream.c_avail_out)
More information about the pypy-commit
mailing list