[Python-checkins] r68165 - in python/branches/py3k: Misc/NEWS Modules/zlibmodule.c
antoine.pitrou
python-checkins at python.org
Fri Jan 2 18:34:35 CET 2009
Author: antoine.pitrou
Date: Fri Jan 2 18:34:35 2009
New Revision: 68165
Log:
Issue #4738: finer-grained locking in the zlib module.
Modified:
python/branches/py3k/Misc/NEWS
python/branches/py3k/Modules/zlibmodule.c
Modified: python/branches/py3k/Misc/NEWS
==============================================================================
--- python/branches/py3k/Misc/NEWS (original)
+++ python/branches/py3k/Misc/NEWS Fri Jan 2 18:34:35 2009
@@ -175,6 +175,10 @@
Extension Modules
-----------------
+- Issue #4738: Each zlib object now has a separate lock, allowing to compress
+ or decompress several streams at once on multi-CPU systems. Also, the GIL
+ is now released when computing the CRC of a large buffer. Patch by ebfe.
+
- Issue #1040026: Fix os.times result on systems where HZ is incorrect.
- Issues #3167, #3682: Fix test_math failures for log, log10 on Solaris,
Modified: python/branches/py3k/Modules/zlibmodule.c
==============================================================================
--- python/branches/py3k/Modules/zlibmodule.c (original)
+++ python/branches/py3k/Modules/zlibmodule.c Fri Jan 2 18:34:35 2009
@@ -9,38 +9,15 @@
#include "zlib.h"
#ifdef WITH_THREAD
-#include "pythread.h"
-
-/* #defs ripped off from _tkinter.c, even though the situation here is much
- simpler, because we don't have to worry about waiting for Tcl
- events! And, since zlib itself is threadsafe, we don't need to worry
- about re-entering zlib functions.
-
- N.B.
-
- Since ENTER_ZLIB and LEAVE_ZLIB only need to be called on functions
- that modify the components of preexisting de/compress objects, it
- could prove to be a performance gain on multiprocessor machines if
- there was an de/compress object-specific lock. However, for the
- moment the ENTER_ZLIB and LEAVE_ZLIB calls are global for ALL
- de/compress objects.
- */
-
-static PyThread_type_lock zlib_lock = NULL; /* initialized on module load */
-
-#define ENTER_ZLIB \
- Py_BEGIN_ALLOW_THREADS \
- PyThread_acquire_lock(zlib_lock, 1); \
- Py_END_ALLOW_THREADS
-
-#define LEAVE_ZLIB \
- PyThread_release_lock(zlib_lock);
-
+ #include "pythread.h"
+ #define ENTER_ZLIB(obj) \
+ Py_BEGIN_ALLOW_THREADS; \
+ PyThread_acquire_lock((obj)->lock, 1); \
+ Py_END_ALLOW_THREADS;
+ #define LEAVE_ZLIB(obj) PyThread_release_lock((obj)->lock);
#else
-
-#define ENTER_ZLIB
-#define LEAVE_ZLIB
-
+ #define ENTER_ZLIB(obj)
+ #define LEAVE_ZLIB(obj)
#endif
/* The following parameters are copied from zutil.h, version 0.95 */
@@ -67,6 +44,9 @@
PyObject *unused_data;
PyObject *unconsumed_tail;
int is_initialised;
+ #ifdef WITH_THREAD
+ PyThread_type_lock lock;
+ #endif
} compobject;
static void
@@ -106,6 +86,9 @@
Py_DECREF(self);
return NULL;
}
+#ifdef WITH_THREAD
+ self->lock = PyThread_allocate_lock();
+#endif
return self;
}
@@ -376,23 +359,30 @@
}
static void
-Comp_dealloc(compobject *self)
+Dealloc(compobject *self)
{
- if (self->is_initialised)
- deflateEnd(&self->zst);
+#ifdef WITH_THREAD
+ PyThread_free_lock(self->lock);
+#endif
Py_XDECREF(self->unused_data);
Py_XDECREF(self->unconsumed_tail);
PyObject_Del(self);
}
static void
+Comp_dealloc(compobject *self)
+{
+ if (self->is_initialised)
+ deflateEnd(&self->zst);
+ Dealloc(self);
+}
+
+static void
Decomp_dealloc(compobject *self)
{
if (self->is_initialised)
- inflateEnd(&self->zst);
- Py_XDECREF(self->unused_data);
- Py_XDECREF(self->unconsumed_tail);
- PyObject_Del(self);
+ inflateEnd(&self->zst);
+ Dealloc(self);
}
PyDoc_STRVAR(comp_compress__doc__,
@@ -422,7 +412,7 @@
return NULL;
}
- ENTER_ZLIB
+ ENTER_ZLIB(self);
start_total_out = self->zst.total_out;
self->zst.avail_in = inplen;
@@ -468,7 +458,7 @@
}
error:
- LEAVE_ZLIB
+ LEAVE_ZLIB(self);
PyBuffer_Release(&pinput);
return RetVal;
}
@@ -514,7 +504,7 @@
return NULL;
}
- ENTER_ZLIB
+ ENTER_ZLIB(self);
start_total_out = self->zst.total_out;
self->zst.avail_in = inplen;
@@ -600,7 +590,7 @@
}
error:
- LEAVE_ZLIB
+ LEAVE_ZLIB(self);
PyBuffer_Release(&pinput);
return RetVal;
}
@@ -633,7 +623,7 @@
if (!(RetVal = PyBytes_FromStringAndSize(NULL, length)))
return NULL;
- ENTER_ZLIB
+ ENTER_ZLIB(self);
start_total_out = self->zst.total_out;
self->zst.avail_in = 0;
@@ -693,7 +683,7 @@
}
error:
- LEAVE_ZLIB
+ LEAVE_ZLIB(self);
return RetVal;
}
@@ -714,7 +704,7 @@
/* Copy the zstream state
* We use ENTER_ZLIB / LEAVE_ZLIB to make this thread-safe
*/
- ENTER_ZLIB
+ ENTER_ZLIB(self);
err = deflateCopy(&retval->zst, &self->zst);
switch(err) {
case(Z_OK):
@@ -730,7 +720,6 @@
zlib_error(self->zst, err, "while copying compression object");
goto error;
}
-
Py_INCREF(self->unused_data);
Py_INCREF(self->unconsumed_tail);
Py_XDECREF(retval->unused_data);
@@ -741,11 +730,11 @@
/* Mark it as being initialized */
retval->is_initialised = 1;
- LEAVE_ZLIB
+ LEAVE_ZLIB(self);
return (PyObject *)retval;
error:
- LEAVE_ZLIB
+ LEAVE_ZLIB(self);
Py_XDECREF(retval);
return NULL;
}
@@ -765,7 +754,7 @@
/* Copy the zstream state
* We use ENTER_ZLIB / LEAVE_ZLIB to make this thread-safe
*/
- ENTER_ZLIB
+ ENTER_ZLIB(self);
err = inflateCopy(&retval->zst, &self->zst);
switch(err) {
case(Z_OK):
@@ -792,11 +781,11 @@
/* Mark it as being initialized */
retval->is_initialised = 1;
- LEAVE_ZLIB
+ LEAVE_ZLIB(self);
return (PyObject *)retval;
error:
- LEAVE_ZLIB
+ LEAVE_ZLIB(self);
Py_XDECREF(retval);
return NULL;
}
@@ -826,7 +815,7 @@
return NULL;
- ENTER_ZLIB
+ ENTER_ZLIB(self);
start_total_out = self->zst.total_out;
self->zst.avail_out = length;
@@ -873,7 +862,7 @@
error:
- LEAVE_ZLIB
+ LEAVE_ZLIB(self);
return retval;
}
@@ -921,12 +910,20 @@
PyZlib_adler32(PyObject *self, PyObject *args)
{
unsigned int adler32val = 1; /* adler32(0L, Z_NULL, 0) */
- Byte *buf;
- int len;
+ Py_buffer pbuf;
- if (!PyArg_ParseTuple(args, "s#|I:adler32", &buf, &len, &adler32val))
+ if (!PyArg_ParseTuple(args, "s*|I:adler32", &pbuf, &adler32val))
return NULL;
- adler32val = adler32(adler32val, buf, len);
+ /* Releasing the GIL for very small buffers is inefficient
+ and may lower performance */
+ if (pbuf.len > 1024*5) {
+ Py_BEGIN_ALLOW_THREADS
+ adler32val = adler32(adler32val, pbuf.buf, pbuf.len);
+ Py_END_ALLOW_THREADS
+ } else {
+ adler32val = adler32(adler32val, pbuf.buf, pbuf.len);
+ }
+ PyBuffer_Release(&pbuf);
return PyLong_FromUnsignedLong(adler32val & 0xffffffffU);
}
@@ -945,7 +942,15 @@
if (!PyArg_ParseTuple(args, "s*|I:crc32", &pbuf, &crc32val))
return NULL;
- signed_val = crc32(crc32val, pbuf.buf, pbuf.len);
+ /* Releasing the GIL for very small buffers is inefficient
+ and may lower performance */
+ if (pbuf.len > 1024*5) {
+ Py_BEGIN_ALLOW_THREADS
+ signed_val = crc32(crc32val, pbuf.buf, pbuf.len);
+ Py_END_ALLOW_THREADS
+ } else {
+ signed_val = crc32(crc32val, pbuf.buf, pbuf.len);
+ }
PyBuffer_Release(&pbuf);
return PyLong_FromUnsignedLong(signed_val & 0xffffffffU);
}
@@ -1096,8 +1101,5 @@
PyModule_AddStringConstant(m, "__version__", "1.0");
-#ifdef WITH_THREAD
- zlib_lock = PyThread_allocate_lock();
-#endif /* WITH_THREAD */
return m;
}
More information about the Python-checkins
mailing list