[Python-checkins] cpython: Close #18294: Fix the zlib module to make it 64-bit safe
victor.stinner
python-checkins at python.org
Thu Nov 21 22:38:51 CET 2013
http://hg.python.org/cpython/rev/f947fe289db8
changeset: 87319:f947fe289db8
user: Victor Stinner <victor.stinner at gmail.com>
date: Thu Nov 21 22:33:21 2013 +0100
summary:
Close #18294: Fix the zlib module to make it 64-bit safe
files:
Misc/NEWS | 2 +
Modules/zlibmodule.c | 180 +++++++++++++++++++++---------
2 files changed, 127 insertions(+), 55 deletions(-)
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -59,6 +59,8 @@
Library
-------
+- Issue #18294: Fix the zlib module to make it 64-bit safe.
+
- Issue #19682: Fix compatibility issue with old version of OpenSSL that
was introduced by Issue #18379.
diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c
--- a/Modules/zlibmodule.c
+++ b/Modules/zlibmodule.c
@@ -176,7 +176,7 @@
if (!PyArg_ParseTuple(args, "y*|i:compress", &pinput, &level))
return NULL;
- if (pinput.len > UINT_MAX) {
+ if ((size_t)pinput.len > UINT_MAX) {
PyErr_SetString(PyExc_OverflowError,
"Size does not fit in an unsigned int");
goto error;
@@ -245,6 +245,45 @@
return ReturnVal;
}
+/*[python]
+
+class uint_converter(CConverter):
+ type = 'unsigned int'
+ converter = 'uint_converter'
+
+[python]*/
+/*[python checksum: da39a3ee5e6b4b0d3255bfef95601890afd80709]*/
+
+static int
+uint_converter(PyObject *obj, void *ptr)
+{
+ long val;
+ unsigned long uval;
+
+ val = PyLong_AsLong(obj);
+ if (val == -1 && PyErr_Occurred()) {
+ uval = PyLong_AsUnsignedLong(obj);
+ if (uval == (unsigned long)-1 && PyErr_Occurred())
+ return 0;
+ if (uval > UINT_MAX) {
+ PyErr_SetString(PyExc_OverflowError,
+ "Python int too large for C unsigned int");
+ return 0;
+ }
+ }
+ else {
+ if (val < 0) {
+ PyErr_SetString(PyExc_ValueError,
+ "value must be positive");
+ return 0;
+ }
+ uval = (unsigned long)val;
+ }
+
+ *(unsigned int *)ptr = Py_SAFE_DOWNCAST(uval, unsigned long, unsigned int);
+ return 1;
+}
+
PyDoc_STRVAR(decompress__doc__,
"decompress(string[, wbits[, bufsize]]) -- Return decompressed string.\n"
"\n"
@@ -260,14 +299,14 @@
unsigned int length;
int err;
int wsize=DEF_WBITS;
- Py_ssize_t r_strlen=DEFAULTALLOC;
+ unsigned int bufsize = DEFAULTALLOC, new_bufsize;
z_stream zst;
- if (!PyArg_ParseTuple(args, "y*|in:decompress",
- &pinput, &wsize, &r_strlen))
+ if (!PyArg_ParseTuple(args, "y*|iO&:decompress",
+ &pinput, &wsize, uint_converter, &bufsize))
return NULL;
- if (pinput.len > UINT_MAX) {
+ if ((size_t)pinput.len > UINT_MAX) {
PyErr_SetString(PyExc_OverflowError,
"Size does not fit in an unsigned int");
goto error;
@@ -275,13 +314,13 @@
input = pinput.buf;
length = (unsigned int)pinput.len;
- if (r_strlen <= 0)
- r_strlen = 1;
+ if (bufsize == 0)
+ bufsize = 1;
zst.avail_in = length;
- zst.avail_out = r_strlen;
+ zst.avail_out = bufsize;
- if (!(result_str = PyBytes_FromStringAndSize(NULL, r_strlen)))
+ if (!(result_str = PyBytes_FromStringAndSize(NULL, bufsize)))
goto error;
zst.opaque = NULL;
@@ -326,14 +365,18 @@
/* fall through */
case(Z_OK):
/* need more memory */
- if (_PyBytes_Resize(&result_str, r_strlen << 1) < 0) {
+ if (bufsize <= (UINT_MAX >> 1))
+ new_bufsize = bufsize << 1;
+ else
+ new_bufsize = UINT_MAX;
+ if (_PyBytes_Resize(&result_str, new_bufsize) < 0) {
inflateEnd(&zst);
goto error;
}
zst.next_out =
- (unsigned char *)PyBytes_AS_STRING(result_str) + r_strlen;
- zst.avail_out = r_strlen;
- r_strlen = r_strlen << 1;
+ (unsigned char *)PyBytes_AS_STRING(result_str) + bufsize;
+ zst.avail_out = bufsize;
+ bufsize = new_bufsize;
break;
default:
inflateEnd(&zst);
@@ -363,7 +406,7 @@
static PyObject *
PyZlib_compressobj(PyObject *selfptr, PyObject *args, PyObject *kwargs)
{
- compobject *self;
+ compobject *self = NULL;
int level=Z_DEFAULT_COMPRESSION, method=DEFLATED;
int wbits=MAX_WBITS, memLevel=DEF_MEM_LEVEL, strategy=0, err;
Py_buffer zdict;
@@ -376,6 +419,12 @@
&memLevel, &strategy, &zdict))
return NULL;
+ if (zdict.buf != NULL && (size_t)zdict.len > UINT_MAX) {
+ PyErr_SetString(PyExc_OverflowError,
+ "zdict length does not fit in an unsigned int");
+ goto error;
+ }
+
self = newcompobject(&Comptype);
if (self==NULL)
goto error;
@@ -391,7 +440,8 @@
if (zdict.buf == NULL) {
goto success;
} else {
- err = deflateSetDictionary(&self->zst, zdict.buf, zdict.len);
+ err = deflateSetDictionary(&self->zst,
+ zdict.buf, (unsigned int)zdict.len);
switch (err) {
case (Z_OK):
goto success;
@@ -515,7 +565,7 @@
{
int err;
unsigned int inplen;
- Py_ssize_t length = DEFAULTALLOC;
+ unsigned int length = DEFAULTALLOC, new_length;
PyObject *RetVal = NULL;
Py_buffer pinput;
Byte *input;
@@ -523,13 +573,13 @@
if (!PyArg_ParseTuple(args, "y*:compress", &pinput))
return NULL;
- if (pinput.len > UINT_MAX) {
+ if ((size_t)pinput.len > UINT_MAX) {
PyErr_SetString(PyExc_OverflowError,
"Size does not fit in an unsigned int");
goto error_outer;
}
input = pinput.buf;
- inplen = pinput.len;
+ inplen = (unsigned int)pinput.len;
if (!(RetVal = PyBytes_FromStringAndSize(NULL, length)))
goto error_outer;
@@ -549,14 +599,18 @@
/* while Z_OK and the output buffer is full, there might be more output,
so extend the output buffer and try again */
while (err == Z_OK && self->zst.avail_out == 0) {
- if (_PyBytes_Resize(&RetVal, length << 1) < 0) {
+ if (length <= (UINT_MAX >> 1))
+ new_length = length << 1;
+ else
+ new_length = UINT_MAX;
+ if (_PyBytes_Resize(&RetVal, new_length) < 0) {
Py_CLEAR(RetVal);
goto error;
}
self->zst.next_out =
(unsigned char *)PyBytes_AS_STRING(RetVal) + length;
self->zst.avail_out = length;
- length = length << 1;
+ length = new_length;
Py_BEGIN_ALLOW_THREADS
err = deflate(&(self->zst), Z_NO_FLUSH);
@@ -596,7 +650,7 @@
Py_ssize_t old_size = PyBytes_GET_SIZE(self->unused_data);
Py_ssize_t new_size;
PyObject *new_data;
- if ((Py_ssize_t)self->zst.avail_in > PY_SSIZE_T_MAX - old_size) {
+ if ((size_t)self->zst.avail_in > (size_t)UINT_MAX - (size_t)old_size) {
PyErr_NoMemory();
return -1;
}
@@ -636,7 +690,7 @@
data: Py_buffer
The binary data to decompress.
- max_length: int = 0
+ max_length: uint = 0
The maximum allowable length of the decompressed data.
Unconsumed input data will be stored in
the unconsumed_tail attribute.
@@ -668,18 +722,18 @@
{"decompress", (PyCFunction)zlib_Decompress_decompress, METH_VARARGS, zlib_Decompress_decompress__doc__},
static PyObject *
-zlib_Decompress_decompress_impl(PyObject *self, Py_buffer *data, int max_length);
+zlib_Decompress_decompress_impl(PyObject *self, Py_buffer *data, unsigned int max_length);
static PyObject *
zlib_Decompress_decompress(PyObject *self, PyObject *args)
{
PyObject *return_value = NULL;
Py_buffer data;
- int max_length = 0;
+ unsigned int max_length = 0;
if (!PyArg_ParseTuple(args,
- "y*|i:decompress",
- &data, &max_length))
+ "y*|O&:decompress",
+ &data, uint_converter, &max_length))
goto exit;
return_value = zlib_Decompress_decompress_impl(self, &data, max_length);
@@ -691,29 +745,20 @@
}
static PyObject *
-zlib_Decompress_decompress_impl(PyObject *self, Py_buffer *data, int max_length)
-/*[clinic checksum: bfac7a0f07e891869d87c665a76dc2611014420f]*/
+zlib_Decompress_decompress_impl(PyObject *self, Py_buffer *data, unsigned int max_length)
+/*[clinic checksum: 76ca9259e3f5ca86bae9da3d0e75637b5d492234]*/
{
compobject *zself = (compobject *)self;
int err;
- unsigned int inplen;
- Py_ssize_t old_length, length = DEFAULTALLOC;
+ unsigned int old_length, length = DEFAULTALLOC;
PyObject *RetVal = NULL;
- Byte *input;
unsigned long start_total_out;
- if (data->len > UINT_MAX) {
+ if ((size_t)data->len > UINT_MAX) {
PyErr_SetString(PyExc_OverflowError,
"Size does not fit in an unsigned int");
return NULL;
}
- input = data->buf;
- inplen = data->len;
- if (max_length < 0) {
- PyErr_SetString(PyExc_ValueError,
- "max_length must be greater than zero");
- return NULL;
- }
/* limit amount of data allocated to max_length */
if (max_length && length > max_length)
@@ -724,8 +769,8 @@
ENTER_ZLIB(zself);
start_total_out = zself->zst.total_out;
- zself->zst.avail_in = inplen;
- zself->zst.next_in = input;
+ zself->zst.avail_in = (unsigned int)data->len;
+ zself->zst.next_in = data->buf;
zself->zst.avail_out = length;
zself->zst.next_out = (unsigned char *)PyBytes_AS_STRING(RetVal);
@@ -740,12 +785,21 @@
RetVal = NULL;
goto error;
}
- err = inflateSetDictionary(&(zself->zst), zdict_buf.buf, zdict_buf.len);
+
+ if ((size_t)zdict_buf.len > UINT_MAX) {
+ PyErr_SetString(PyExc_OverflowError,
+ "zdict length does not fit in an unsigned int");
+ PyBuffer_Release(&zdict_buf);
+ Py_CLEAR(RetVal);
+ goto error;
+ }
+
+ err = inflateSetDictionary(&(zself->zst),
+ zdict_buf.buf, (unsigned int)zdict_buf.len);
PyBuffer_Release(&zdict_buf);
if (err != Z_OK) {
zlib_error(zself->zst, err, "while decompressing data");
- Py_DECREF(RetVal);
- RetVal = NULL;
+ Py_CLEAR(RetVal);
goto error;
}
/* Repeat the call to inflate. */
@@ -824,7 +878,8 @@
static PyObject *
PyZlib_flush(compobject *self, PyObject *args)
{
- int err, length = DEFAULTALLOC;
+ int err;
+ unsigned int length = DEFAULTALLOC, new_length;
PyObject *RetVal;
int flushmode = Z_FINISH;
unsigned long start_total_out;
@@ -855,14 +910,18 @@
/* while Z_OK and the output buffer is full, there might be more output,
so extend the output buffer and try again */
while (err == Z_OK && self->zst.avail_out == 0) {
- if (_PyBytes_Resize(&RetVal, length << 1) < 0) {
+ if (length <= (UINT_MAX >> 1))
+ new_length = length << 1;
+ else
+ new_length = UINT_MAX;
+ if (_PyBytes_Resize(&RetVal, new_length) < 0) {
Py_CLEAR(RetVal);
goto error;
}
self->zst.next_out =
(unsigned char *)PyBytes_AS_STRING(RetVal) + length;
self->zst.avail_out = length;
- length = length << 1;
+ length = new_length;
Py_BEGIN_ALLOW_THREADS
err = deflate(&(self->zst), flushmode);
@@ -1041,24 +1100,31 @@
static PyObject *
PyZlib_unflush(compobject *self, PyObject *args)
{
- int err, length = DEFAULTALLOC;
+ int err;
+ unsigned int length = DEFAULTALLOC, new_length;
PyObject * retval = NULL;
unsigned long start_total_out;
+ Py_ssize_t size;
- if (!PyArg_ParseTuple(args, "|i:flush", &length))
+ if (!PyArg_ParseTuple(args, "|O&:flush", uint_converter, &length))
return NULL;
- if (length <= 0) {
+ if (length == 0) {
PyErr_SetString(PyExc_ValueError, "length must be greater than zero");
return NULL;
}
+
if (!(retval = PyBytes_FromStringAndSize(NULL, length)))
return NULL;
ENTER_ZLIB(self);
+ size = PyBytes_GET_SIZE(self->unconsumed_tail);
+
start_total_out = self->zst.total_out;
- self->zst.avail_in = PyBytes_GET_SIZE(self->unconsumed_tail);
+ /* save_unconsumed_input() ensures that unconsumed_tail length is lesser
+ or equal than UINT_MAX */
+ self->zst.avail_in = Py_SAFE_DOWNCAST(size, Py_ssize_t, unsigned int);
self->zst.next_in = (Byte *)PyBytes_AS_STRING(self->unconsumed_tail);
self->zst.avail_out = length;
self->zst.next_out = (Byte *)PyBytes_AS_STRING(retval);
@@ -1070,13 +1136,17 @@
/* while Z_OK and the output buffer is full, there might be more output,
so extend the output buffer and try again */
while ((err == Z_OK || err == Z_BUF_ERROR) && self->zst.avail_out == 0) {
- if (_PyBytes_Resize(&retval, length << 1) < 0) {
+ if (length <= (UINT_MAX >> 1))
+ new_length = length << 1;
+ else
+ new_length = UINT_MAX;
+ if (_PyBytes_Resize(&retval, new_length) < 0) {
Py_CLEAR(retval);
goto error;
}
self->zst.next_out = (Byte *)PyBytes_AS_STRING(retval) + length;
self->zst.avail_out = length;
- length = length << 1;
+ length = new_length;
Py_BEGIN_ALLOW_THREADS
err = inflate(&(self->zst), Z_FINISH);
@@ -1168,7 +1238,7 @@
Py_BEGIN_ALLOW_THREADS
/* Avoid truncation of length for very large buffers. adler32() takes
length as an unsigned int, which may be narrower than Py_ssize_t. */
- while (len > (size_t) UINT_MAX) {
+ while ((size_t)len > UINT_MAX) {
adler32val = adler32(adler32val, buf, UINT_MAX);
buf += (size_t) UINT_MAX;
len -= (size_t) UINT_MAX;
@@ -1206,7 +1276,7 @@
Py_BEGIN_ALLOW_THREADS
/* Avoid truncation of length for very large buffers. crc32() takes
length as an unsigned int, which may be narrower than Py_ssize_t. */
- while (len > (size_t) UINT_MAX) {
+ while ((size_t)len > UINT_MAX) {
crc32val = crc32(crc32val, buf, UINT_MAX);
buf += (size_t) UINT_MAX;
len -= (size_t) UINT_MAX;
--
Repository URL: http://hg.python.org/cpython
More information about the Python-checkins
mailing list