mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-11-01 02:38:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 15

Issue #4738: finer-grained locking in the zlib module.

Browse source
This commit is contained in:
Antoine Pitrou 2009-01-02 17:34:35 +00:00
parent e9f8bf023a
commit 31f30b17fe
2 changed files with 67 additions and 61 deletions
Download patch file Download diff file Expand all files Collapse all files

4
Misc/NEWS
View file

@ -175,6 +175,10 @@ Tools/Demos
Extension Modules Extension Modules
----------------- -----------------
- Issue #4738: Each zlib object now has a separate lock, allowing to compress
or decompress several streams at once on multi-CPU systems. Also, the GIL
is now released when computing the CRC of a large buffer. Patch by ebfe.
- Issue #1040026: Fix os.times result on systems where HZ is incorrect. - Issue #1040026: Fix os.times result on systems where HZ is incorrect.
- Issues #3167, #3682: Fix test_math failures for log, log10 on Solaris, - Issues #3167, #3682: Fix test_math failures for log, log10 on Solaris,

124
Modules/zlibmodule.c
View file

@ -9,38 +9,15 @@
#include "zlib.h" #include "zlib.h"
#ifdef WITH_THREAD #ifdef WITH_THREAD
#include "pythread.h" #include "pythread.h"
#define ENTER_ZLIB(obj) \
/* #defs ripped off from _tkinter.c, even though the situation here is much Py_BEGIN_ALLOW_THREADS; \
simpler, because we don't have to worry about waiting for Tcl PyThread_acquire_lock((obj)->lock, 1); \
events! And, since zlib itself is threadsafe, we don't need to worry Py_END_ALLOW_THREADS;
about re-entering zlib functions. #define LEAVE_ZLIB(obj) PyThread_release_lock((obj)->lock);
N.B.
Since ENTER_ZLIB and LEAVE_ZLIB only need to be called on functions
that modify the components of preexisting de/compress objects, it
could prove to be a performance gain on multiprocessor machines if
there was an de/compress object-specific lock. However, for the
moment the ENTER_ZLIB and LEAVE_ZLIB calls are global for ALL
de/compress objects.
*/
static PyThread_type_lock zlib_lock = NULL; /* initialized on module load */
#define ENTER_ZLIB \
Py_BEGIN_ALLOW_THREADS \
PyThread_acquire_lock(zlib_lock, 1); \
Py_END_ALLOW_THREADS
#define LEAVE_ZLIB \
PyThread_release_lock(zlib_lock);
#else #else
#define ENTER_ZLIB(obj)
#define ENTER_ZLIB #define LEAVE_ZLIB(obj)
#define LEAVE_ZLIB
#endif #endif
/* The following parameters are copied from zutil.h, version 0.95 */ /* The following parameters are copied from zutil.h, version 0.95 */
@ -67,6 +44,9 @@ typedef struct
PyObject *unused_data; PyObject *unused_data;
PyObject *unconsumed_tail; PyObject *unconsumed_tail;
int is_initialised; int is_initialised;
#ifdef WITH_THREAD
PyThread_type_lock lock;
#endif
} compobject; } compobject;
static void static void
@ -106,6 +86,9 @@ newcompobject(PyTypeObject *type)
Py_DECREF(self); Py_DECREF(self);
return NULL; return NULL;
} }
#ifdef WITH_THREAD
self->lock = PyThread_allocate_lock();
#endif
return self; return self;
} }
@ -376,23 +359,30 @@ PyZlib_decompressobj(PyObject *selfptr, PyObject *args)
} }
static void static void
Comp_dealloc(compobject *self) Dealloc(compobject *self)
{ {
if (self->is_initialised) #ifdef WITH_THREAD
deflateEnd(&self->zst); PyThread_free_lock(self->lock);
#endif
Py_XDECREF(self->unused_data); Py_XDECREF(self->unused_data);
Py_XDECREF(self->unconsumed_tail); Py_XDECREF(self->unconsumed_tail);
PyObject_Del(self); PyObject_Del(self);
} }
static void
Comp_dealloc(compobject *self)
{
if (self->is_initialised)
deflateEnd(&self->zst);
Dealloc(self);
}
static void static void
Decomp_dealloc(compobject *self) Decomp_dealloc(compobject *self)
{ {
if (self->is_initialised) if (self->is_initialised)
inflateEnd(&self->zst); inflateEnd(&self->zst);
Py_XDECREF(self->unused_data); Dealloc(self);
Py_XDECREF(self->unconsumed_tail);
PyObject_Del(self);
} }
PyDoc_STRVAR(comp_compress__doc__, PyDoc_STRVAR(comp_compress__doc__,
@ -422,7 +412,7 @@ PyZlib_objcompress(compobject *self, PyObject *args)
return NULL; return NULL;
} }
ENTER_ZLIB ENTER_ZLIB(self);
start_total_out = self->zst.total_out; start_total_out = self->zst.total_out;
self->zst.avail_in = inplen; self->zst.avail_in = inplen;
@ -468,7 +458,7 @@ PyZlib_objcompress(compobject *self, PyObject *args)
} }
error: error:
LEAVE_ZLIB LEAVE_ZLIB(self);
PyBuffer_Release(&pinput); PyBuffer_Release(&pinput);
return RetVal; return RetVal;
} }
@ -514,7 +504,7 @@ PyZlib_objdecompress(compobject *self, PyObject *args)
return NULL; return NULL;
} }
ENTER_ZLIB ENTER_ZLIB(self);
start_total_out = self->zst.total_out; start_total_out = self->zst.total_out;
self->zst.avail_in = inplen; self->zst.avail_in = inplen;
@ -600,7 +590,7 @@ PyZlib_objdecompress(compobject *self, PyObject *args)
} }
error: error:
LEAVE_ZLIB LEAVE_ZLIB(self);
PyBuffer_Release(&pinput); PyBuffer_Release(&pinput);
return RetVal; return RetVal;
} }
@ -633,7 +623,7 @@ PyZlib_flush(compobject *self, PyObject *args)
if (!(RetVal = PyBytes_FromStringAndSize(NULL, length))) if (!(RetVal = PyBytes_FromStringAndSize(NULL, length)))
return NULL; return NULL;
ENTER_ZLIB ENTER_ZLIB(self);
start_total_out = self->zst.total_out; start_total_out = self->zst.total_out;
self->zst.avail_in = 0; self->zst.avail_in = 0;
@ -693,7 +683,7 @@ PyZlib_flush(compobject *self, PyObject *args)
} }
error: error:
LEAVE_ZLIB LEAVE_ZLIB(self);
return RetVal; return RetVal;
} }
@ -714,7 +704,7 @@ PyZlib_copy(compobject *self)
/* Copy the zstream state /* Copy the zstream state
* We use ENTER_ZLIB / LEAVE_ZLIB to make this thread-safe * We use ENTER_ZLIB / LEAVE_ZLIB to make this thread-safe
*/ */
ENTER_ZLIB ENTER_ZLIB(self);
err = deflateCopy(&retval->zst, &self->zst); err = deflateCopy(&retval->zst, &self->zst);
switch(err) { switch(err) {
case(Z_OK): case(Z_OK):
@ -730,7 +720,6 @@ PyZlib_copy(compobject *self)
zlib_error(self->zst, err, "while copying compression object"); zlib_error(self->zst, err, "while copying compression object");
goto error; goto error;
} }
Py_INCREF(self->unused_data); Py_INCREF(self->unused_data);
Py_INCREF(self->unconsumed_tail); Py_INCREF(self->unconsumed_tail);
Py_XDECREF(retval->unused_data); Py_XDECREF(retval->unused_data);
@ -741,11 +730,11 @@ PyZlib_copy(compobject *self)
/* Mark it as being initialized */ /* Mark it as being initialized */
retval->is_initialised = 1; retval->is_initialised = 1;
LEAVE_ZLIB LEAVE_ZLIB(self);
return (PyObject *)retval; return (PyObject *)retval;
error: error:
LEAVE_ZLIB LEAVE_ZLIB(self);
Py_XDECREF(retval); Py_XDECREF(retval);
return NULL; return NULL;
} }
@ -765,7 +754,7 @@ PyZlib_uncopy(compobject *self)
/* Copy the zstream state /* Copy the zstream state
* We use ENTER_ZLIB / LEAVE_ZLIB to make this thread-safe * We use ENTER_ZLIB / LEAVE_ZLIB to make this thread-safe
*/ */
ENTER_ZLIB ENTER_ZLIB(self);
err = inflateCopy(&retval->zst, &self->zst); err = inflateCopy(&retval->zst, &self->zst);
switch(err) { switch(err) {
case(Z_OK): case(Z_OK):
@ -792,11 +781,11 @@ PyZlib_uncopy(compobject *self)
/* Mark it as being initialized */ /* Mark it as being initialized */
retval->is_initialised = 1; retval->is_initialised = 1;
LEAVE_ZLIB LEAVE_ZLIB(self);
return (PyObject *)retval; return (PyObject *)retval;
error: error:
LEAVE_ZLIB LEAVE_ZLIB(self);
Py_XDECREF(retval); Py_XDECREF(retval);
return NULL; return NULL;
} }
@ -826,7 +815,7 @@ PyZlib_unflush(compobject *self, PyObject *args)
return NULL; return NULL;
ENTER_ZLIB ENTER_ZLIB(self);
start_total_out = self->zst.total_out; start_total_out = self->zst.total_out;
self->zst.avail_out = length; self->zst.avail_out = length;
@ -873,7 +862,7 @@ PyZlib_unflush(compobject *self, PyObject *args)
error: error:
LEAVE_ZLIB LEAVE_ZLIB(self);
return retval; return retval;
} }
@ -921,12 +910,20 @@ static PyObject *
PyZlib_adler32(PyObject *self, PyObject *args) PyZlib_adler32(PyObject *self, PyObject *args)
{ {
unsigned int adler32val = 1; /* adler32(0L, Z_NULL, 0) */ unsigned int adler32val = 1; /* adler32(0L, Z_NULL, 0) */
Byte *buf; Py_buffer pbuf;
int len;
if (!PyArg_ParseTuple(args, "s#|I:adler32", &buf, &len, &adler32val)) if (!PyArg_ParseTuple(args, "s*|I:adler32", &pbuf, &adler32val))
return NULL; return NULL;
adler32val = adler32(adler32val, buf, len); /* Releasing the GIL for very small buffers is inefficient
and may lower performance */
if (pbuf.len > 1024*5) {
Py_BEGIN_ALLOW_THREADS
adler32val = adler32(adler32val, pbuf.buf, pbuf.len);
Py_END_ALLOW_THREADS
} else {
adler32val = adler32(adler32val, pbuf.buf, pbuf.len);
}
PyBuffer_Release(&pbuf);
return PyLong_FromUnsignedLong(adler32val & 0xffffffffU); return PyLong_FromUnsignedLong(adler32val & 0xffffffffU);
} }
@ -945,7 +942,15 @@ PyZlib_crc32(PyObject *self, PyObject *args)
if (!PyArg_ParseTuple(args, "s*|I:crc32", &pbuf, &crc32val)) if (!PyArg_ParseTuple(args, "s*|I:crc32", &pbuf, &crc32val))
return NULL; return NULL;
signed_val = crc32(crc32val, pbuf.buf, pbuf.len); /* Releasing the GIL for very small buffers is inefficient
and may lower performance */
if (pbuf.len > 1024*5) {
Py_BEGIN_ALLOW_THREADS
signed_val = crc32(crc32val, pbuf.buf, pbuf.len);
Py_END_ALLOW_THREADS
} else {
signed_val = crc32(crc32val, pbuf.buf, pbuf.len);
}
PyBuffer_Release(&pbuf); PyBuffer_Release(&pbuf);
return PyLong_FromUnsignedLong(signed_val & 0xffffffffU); return PyLong_FromUnsignedLong(signed_val & 0xffffffffU);
} }
@ -1096,8 +1101,5 @@ PyInit_zlib(void)
PyModule_AddStringConstant(m, "__version__", "1.0"); PyModule_AddStringConstant(m, "__version__", "1.0");
#ifdef WITH_THREAD
zlib_lock = PyThread_allocate_lock();
#endif /* WITH_THREAD */
return m; return m;
} }