mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-11-04 11:49:12 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 11

Close #18294: Fix the zlib module to make it 64-bit safe

Browse source
This commit is contained in:
Victor Stinner 2013-11-21 22:33:21 +01:00
parent 8820c239f7
commit e079eddf21
2 changed files with 127 additions and 55 deletions
Download patch file Download diff file Expand all files Collapse all files

2
Misc/NEWS
View file

@ -59,6 +59,8 @@ Core and Builtins
Library Library
------- -------
- Issue #18294: Fix the zlib module to make it 64-bit safe.
- Issue #19682: Fix compatibility issue with old version of OpenSSL that - Issue #19682: Fix compatibility issue with old version of OpenSSL that
was introduced by Issue #18379. was introduced by Issue #18379.

180
Modules/zlibmodule.c
View file

@ -176,7 +176,7 @@ PyZlib_compress(PyObject *self, PyObject *args)
if (!PyArg_ParseTuple(args, "y*|i:compress", &pinput, &level)) if (!PyArg_ParseTuple(args, "y*|i:compress", &pinput, &level))
return NULL; return NULL;
if (pinput.len > UINT_MAX) { if ((size_t)pinput.len > UINT_MAX) {
PyErr_SetString(PyExc_OverflowError, PyErr_SetString(PyExc_OverflowError,
"Size does not fit in an unsigned int"); "Size does not fit in an unsigned int");
goto error; goto error;
@ -245,6 +245,45 @@ PyZlib_compress(PyObject *self, PyObject *args)
return ReturnVal; return ReturnVal;
} }
/*[python]
class uint_converter(CConverter):
type = 'unsigned int'
converter = 'uint_converter'
[python]*/
/*[python checksum: da39a3ee5e6b4b0d3255bfef95601890afd80709]*/
static int
uint_converter(PyObject *obj, void *ptr)
{
long val;
unsigned long uval;
val = PyLong_AsLong(obj);
if (val == -1 && PyErr_Occurred()) {
uval = PyLong_AsUnsignedLong(obj);
if (uval == (unsigned long)-1 && PyErr_Occurred())
return 0;
if (uval > UINT_MAX) {
PyErr_SetString(PyExc_OverflowError,
"Python int too large for C unsigned int");
return 0;
}
}
else {
if (val < 0) {
PyErr_SetString(PyExc_ValueError,
"value must be positive");
return 0;
}
uval = (unsigned long)val;
}
*(unsigned int *)ptr = Py_SAFE_DOWNCAST(uval, unsigned long, unsigned int);
return 1;
}
PyDoc_STRVAR(decompress__doc__, PyDoc_STRVAR(decompress__doc__,
"decompress(string[, wbits[, bufsize]]) -- Return decompressed string.\n" "decompress(string[, wbits[, bufsize]]) -- Return decompressed string.\n"
"\n" "\n"
@ -260,14 +299,14 @@ PyZlib_decompress(PyObject *self, PyObject *args)
unsigned int length; unsigned int length;
int err; int err;
int wsize=DEF_WBITS; int wsize=DEF_WBITS;
Py_ssize_t r_strlen=DEFAULTALLOC; unsigned int bufsize = DEFAULTALLOC, new_bufsize;
z_stream zst; z_stream zst;
if (!PyArg_ParseTuple(args, "y*|in:decompress", if (!PyArg_ParseTuple(args, "y*|iO&:decompress",
&pinput, &wsize, &r_strlen)) &pinput, &wsize, uint_converter, &bufsize))
return NULL; return NULL;
if (pinput.len > UINT_MAX) { if ((size_t)pinput.len > UINT_MAX) {
PyErr_SetString(PyExc_OverflowError, PyErr_SetString(PyExc_OverflowError,
"Size does not fit in an unsigned int"); "Size does not fit in an unsigned int");
goto error; goto error;
@ -275,13 +314,13 @@ PyZlib_decompress(PyObject *self, PyObject *args)
input = pinput.buf; input = pinput.buf;
length = (unsigned int)pinput.len; length = (unsigned int)pinput.len;
if (r_strlen <= 0) if (bufsize == 0)
r_strlen = 1; bufsize = 1;
zst.avail_in = length; zst.avail_in = length;
zst.avail_out = r_strlen; zst.avail_out = bufsize;
if (!(result_str = PyBytes_FromStringAndSize(NULL, r_strlen))) if (!(result_str = PyBytes_FromStringAndSize(NULL, bufsize)))
goto error; goto error;
zst.opaque = NULL; zst.opaque = NULL;
@ -326,14 +365,18 @@ PyZlib_decompress(PyObject *self, PyObject *args)
/* fall through */ /* fall through */
case(Z_OK): case(Z_OK):
/* need more memory */ /* need more memory */
if (_PyBytes_Resize(&result_str, r_strlen << 1) < 0) { if (bufsize <= (UINT_MAX >> 1))
new_bufsize = bufsize << 1;
else
new_bufsize = UINT_MAX;
if (_PyBytes_Resize(&result_str, new_bufsize) < 0) {
inflateEnd(&zst); inflateEnd(&zst);
goto error; goto error;
} }
zst.next_out = zst.next_out =
(unsigned char *)PyBytes_AS_STRING(result_str) + r_strlen; (unsigned char *)PyBytes_AS_STRING(result_str) + bufsize;
zst.avail_out = r_strlen; zst.avail_out = bufsize;
r_strlen = r_strlen << 1; bufsize = new_bufsize;
break; break;
default: default:
inflateEnd(&zst); inflateEnd(&zst);
@ -363,7 +406,7 @@ PyZlib_decompress(PyObject *self, PyObject *args)
static PyObject * static PyObject *
PyZlib_compressobj(PyObject *selfptr, PyObject *args, PyObject *kwargs) PyZlib_compressobj(PyObject *selfptr, PyObject *args, PyObject *kwargs)
{ {
compobject *self; compobject *self = NULL;
int level=Z_DEFAULT_COMPRESSION, method=DEFLATED; int level=Z_DEFAULT_COMPRESSION, method=DEFLATED;
int wbits=MAX_WBITS, memLevel=DEF_MEM_LEVEL, strategy=0, err; int wbits=MAX_WBITS, memLevel=DEF_MEM_LEVEL, strategy=0, err;
Py_buffer zdict; Py_buffer zdict;
@ -376,6 +419,12 @@ PyZlib_compressobj(PyObject *selfptr, PyObject *args, PyObject *kwargs)
&memLevel, &strategy, &zdict)) &memLevel, &strategy, &zdict))
return NULL; return NULL;
if (zdict.buf != NULL && (size_t)zdict.len > UINT_MAX) {
PyErr_SetString(PyExc_OverflowError,
"zdict length does not fit in an unsigned int");
goto error;
}
self = newcompobject(&Comptype); self = newcompobject(&Comptype);
if (self==NULL) if (self==NULL)
goto error; goto error;
@ -391,7 +440,8 @@ PyZlib_compressobj(PyObject *selfptr, PyObject *args, PyObject *kwargs)
if (zdict.buf == NULL) { if (zdict.buf == NULL) {
goto success; goto success;
} else { } else {
err = deflateSetDictionary(&self->zst, zdict.buf, zdict.len); err = deflateSetDictionary(&self->zst,
zdict.buf, (unsigned int)zdict.len);
switch (err) { switch (err) {
case (Z_OK): case (Z_OK):
goto success; goto success;
@ -515,7 +565,7 @@ PyZlib_objcompress(compobject *self, PyObject *args)
{ {
int err; int err;
unsigned int inplen; unsigned int inplen;
Py_ssize_t length = DEFAULTALLOC; unsigned int length = DEFAULTALLOC, new_length;
PyObject *RetVal = NULL; PyObject *RetVal = NULL;
Py_buffer pinput; Py_buffer pinput;
Byte *input; Byte *input;
@ -523,13 +573,13 @@ PyZlib_objcompress(compobject *self, PyObject *args)
if (!PyArg_ParseTuple(args, "y*:compress", &pinput)) if (!PyArg_ParseTuple(args, "y*:compress", &pinput))
return NULL; return NULL;
if (pinput.len > UINT_MAX) { if ((size_t)pinput.len > UINT_MAX) {
PyErr_SetString(PyExc_OverflowError, PyErr_SetString(PyExc_OverflowError,
"Size does not fit in an unsigned int"); "Size does not fit in an unsigned int");
goto error_outer; goto error_outer;
} }
input = pinput.buf; input = pinput.buf;
inplen = pinput.len; inplen = (unsigned int)pinput.len;
if (!(RetVal = PyBytes_FromStringAndSize(NULL, length))) if (!(RetVal = PyBytes_FromStringAndSize(NULL, length)))
goto error_outer; goto error_outer;
@ -549,14 +599,18 @@ PyZlib_objcompress(compobject *self, PyObject *args)
/* while Z_OK and the output buffer is full, there might be more output, /* while Z_OK and the output buffer is full, there might be more output,
so extend the output buffer and try again */ so extend the output buffer and try again */
while (err == Z_OK && self->zst.avail_out == 0) { while (err == Z_OK && self->zst.avail_out == 0) {
if (_PyBytes_Resize(&RetVal, length << 1) < 0) { if (length <= (UINT_MAX >> 1))
new_length = length << 1;
else
new_length = UINT_MAX;
if (_PyBytes_Resize(&RetVal, new_length) < 0) {
Py_CLEAR(RetVal); Py_CLEAR(RetVal);
goto error; goto error;
} }
self->zst.next_out = self->zst.next_out =
(unsigned char *)PyBytes_AS_STRING(RetVal) + length; (unsigned char *)PyBytes_AS_STRING(RetVal) + length;
self->zst.avail_out = length; self->zst.avail_out = length;
length = length << 1; length = new_length;
Py_BEGIN_ALLOW_THREADS Py_BEGIN_ALLOW_THREADS
err = deflate(&(self->zst), Z_NO_FLUSH); err = deflate(&(self->zst), Z_NO_FLUSH);
@ -596,7 +650,7 @@ save_unconsumed_input(compobject *self, int err)
Py_ssize_t old_size = PyBytes_GET_SIZE(self->unused_data); Py_ssize_t old_size = PyBytes_GET_SIZE(self->unused_data);
Py_ssize_t new_size; Py_ssize_t new_size;
PyObject *new_data; PyObject *new_data;
if ((Py_ssize_t)self->zst.avail_in > PY_SSIZE_T_MAX - old_size) { if ((size_t)self->zst.avail_in > (size_t)UINT_MAX - (size_t)old_size) {
PyErr_NoMemory(); PyErr_NoMemory();
return -1; return -1;
} }
@ -636,7 +690,7 @@ zlib.Decompress.decompress
data: Py_buffer data: Py_buffer
The binary data to decompress. The binary data to decompress.
max_length: int = 0 max_length: uint = 0
The maximum allowable length of the decompressed data. The maximum allowable length of the decompressed data.
Unconsumed input data will be stored in Unconsumed input data will be stored in
the unconsumed_tail attribute. the unconsumed_tail attribute.
@ -668,18 +722,18 @@ PyDoc_STRVAR(zlib_Decompress_decompress__doc__,
{"decompress", (PyCFunction)zlib_Decompress_decompress, METH_VARARGS, zlib_Decompress_decompress__doc__}, {"decompress", (PyCFunction)zlib_Decompress_decompress, METH_VARARGS, zlib_Decompress_decompress__doc__},
static PyObject * static PyObject *
zlib_Decompress_decompress_impl(PyObject *self, Py_buffer *data, int max_length); zlib_Decompress_decompress_impl(PyObject *self, Py_buffer *data, unsigned int max_length);
static PyObject * static PyObject *
zlib_Decompress_decompress(PyObject *self, PyObject *args) zlib_Decompress_decompress(PyObject *self, PyObject *args)
{ {
PyObject *return_value = NULL; PyObject *return_value = NULL;
Py_buffer data; Py_buffer data;
int max_length = 0; unsigned int max_length = 0;
if (!PyArg_ParseTuple(args, if (!PyArg_ParseTuple(args,
"y*|i:decompress", "y*|O&:decompress",
&data, &max_length)) &data, uint_converter, &max_length))
goto exit; goto exit;
return_value = zlib_Decompress_decompress_impl(self, &data, max_length); return_value = zlib_Decompress_decompress_impl(self, &data, max_length);
@ -691,29 +745,20 @@ exit:
} }
static PyObject * static PyObject *
zlib_Decompress_decompress_impl(PyObject *self, Py_buffer *data, int max_length) zlib_Decompress_decompress_impl(PyObject *self, Py_buffer *data, unsigned int max_length)
/*[clinic checksum: bfac7a0f07e891869d87c665a76dc2611014420f]*/ /*[clinic checksum: 76ca9259e3f5ca86bae9da3d0e75637b5d492234]*/
{ {
compobject *zself = (compobject *)self; compobject *zself = (compobject *)self;
int err; int err;
unsigned int inplen; unsigned int old_length, length = DEFAULTALLOC;
Py_ssize_t old_length, length = DEFAULTALLOC;
PyObject *RetVal = NULL; PyObject *RetVal = NULL;
Byte *input;
unsigned long start_total_out; unsigned long start_total_out;
if (data->len > UINT_MAX) { if ((size_t)data->len > UINT_MAX) {
PyErr_SetString(PyExc_OverflowError, PyErr_SetString(PyExc_OverflowError,
"Size does not fit in an unsigned int"); "Size does not fit in an unsigned int");
return NULL; return NULL;
} }
input = data->buf;
inplen = data->len;
if (max_length < 0) {
PyErr_SetString(PyExc_ValueError,
"max_length must be greater than zero");
return NULL;
}
/* limit amount of data allocated to max_length */ /* limit amount of data allocated to max_length */
if (max_length && length > max_length) if (max_length && length > max_length)
@ -724,8 +769,8 @@ zlib_Decompress_decompress_impl(PyObject *self, Py_buffer *data, int max_length)
ENTER_ZLIB(zself); ENTER_ZLIB(zself);
start_total_out = zself->zst.total_out; start_total_out = zself->zst.total_out;
zself->zst.avail_in = inplen; zself->zst.avail_in = (unsigned int)data->len;
zself->zst.next_in = input; zself->zst.next_in = data->buf;
zself->zst.avail_out = length; zself->zst.avail_out = length;
zself->zst.next_out = (unsigned char *)PyBytes_AS_STRING(RetVal); zself->zst.next_out = (unsigned char *)PyBytes_AS_STRING(RetVal);
@ -740,12 +785,21 @@ zlib_Decompress_decompress_impl(PyObject *self, Py_buffer *data, int max_length)
RetVal = NULL; RetVal = NULL;
goto error; goto error;
} }
err = inflateSetDictionary(&(zself->zst), zdict_buf.buf, zdict_buf.len);
if ((size_t)zdict_buf.len > UINT_MAX) {
PyErr_SetString(PyExc_OverflowError,
"zdict length does not fit in an unsigned int");
PyBuffer_Release(&zdict_buf);
Py_CLEAR(RetVal);
goto error;
}
err = inflateSetDictionary(&(zself->zst),
zdict_buf.buf, (unsigned int)zdict_buf.len);
PyBuffer_Release(&zdict_buf); PyBuffer_Release(&zdict_buf);
if (err != Z_OK) { if (err != Z_OK) {
zlib_error(zself->zst, err, "while decompressing data"); zlib_error(zself->zst, err, "while decompressing data");
Py_DECREF(RetVal); Py_CLEAR(RetVal);
RetVal = NULL;
goto error; goto error;
} }
/* Repeat the call to inflate. */ /* Repeat the call to inflate. */
@ -824,7 +878,8 @@ PyDoc_STRVAR(comp_flush__doc__,
static PyObject * static PyObject *
PyZlib_flush(compobject *self, PyObject *args) PyZlib_flush(compobject *self, PyObject *args)
{ {
int err, length = DEFAULTALLOC; int err;
unsigned int length = DEFAULTALLOC, new_length;
PyObject *RetVal; PyObject *RetVal;
int flushmode = Z_FINISH; int flushmode = Z_FINISH;
unsigned long start_total_out; unsigned long start_total_out;
@ -855,14 +910,18 @@ PyZlib_flush(compobject *self, PyObject *args)
/* while Z_OK and the output buffer is full, there might be more output, /* while Z_OK and the output buffer is full, there might be more output,
so extend the output buffer and try again */ so extend the output buffer and try again */
while (err == Z_OK && self->zst.avail_out == 0) { while (err == Z_OK && self->zst.avail_out == 0) {
if (_PyBytes_Resize(&RetVal, length << 1) < 0) { if (length <= (UINT_MAX >> 1))
new_length = length << 1;
else
new_length = UINT_MAX;
if (_PyBytes_Resize(&RetVal, new_length) < 0) {
Py_CLEAR(RetVal); Py_CLEAR(RetVal);
goto error; goto error;
} }
self->zst.next_out = self->zst.next_out =
(unsigned char *)PyBytes_AS_STRING(RetVal) + length; (unsigned char *)PyBytes_AS_STRING(RetVal) + length;
self->zst.avail_out = length; self->zst.avail_out = length;
length = length << 1; length = new_length;
Py_BEGIN_ALLOW_THREADS Py_BEGIN_ALLOW_THREADS
err = deflate(&(self->zst), flushmode); err = deflate(&(self->zst), flushmode);
@ -1041,24 +1100,31 @@ PyDoc_STRVAR(decomp_flush__doc__,
static PyObject * static PyObject *
PyZlib_unflush(compobject *self, PyObject *args) PyZlib_unflush(compobject *self, PyObject *args)
{ {
int err, length = DEFAULTALLOC; int err;
unsigned int length = DEFAULTALLOC, new_length;
PyObject * retval = NULL; PyObject * retval = NULL;
unsigned long start_total_out; unsigned long start_total_out;
Py_ssize_t size;
if (!PyArg_ParseTuple(args, "|i:flush", &length)) if (!PyArg_ParseTuple(args, "|O&:flush", uint_converter, &length))
return NULL; return NULL;
if (length <= 0) { if (length == 0) {
PyErr_SetString(PyExc_ValueError, "length must be greater than zero"); PyErr_SetString(PyExc_ValueError, "length must be greater than zero");
return NULL; return NULL;
} }
if (!(retval = PyBytes_FromStringAndSize(NULL, length))) if (!(retval = PyBytes_FromStringAndSize(NULL, length)))
return NULL; return NULL;
ENTER_ZLIB(self); ENTER_ZLIB(self);
size = PyBytes_GET_SIZE(self->unconsumed_tail);
start_total_out = self->zst.total_out; start_total_out = self->zst.total_out;
self->zst.avail_in = PyBytes_GET_SIZE(self->unconsumed_tail); /* save_unconsumed_input() ensures that unconsumed_tail length is lesser
or equal than UINT_MAX */
self->zst.avail_in = Py_SAFE_DOWNCAST(size, Py_ssize_t, unsigned int);
self->zst.next_in = (Byte *)PyBytes_AS_STRING(self->unconsumed_tail); self->zst.next_in = (Byte *)PyBytes_AS_STRING(self->unconsumed_tail);
self->zst.avail_out = length; self->zst.avail_out = length;
self->zst.next_out = (Byte *)PyBytes_AS_STRING(retval); self->zst.next_out = (Byte *)PyBytes_AS_STRING(retval);
@ -1070,13 +1136,17 @@ PyZlib_unflush(compobject *self, PyObject *args)
/* while Z_OK and the output buffer is full, there might be more output, /* while Z_OK and the output buffer is full, there might be more output,
so extend the output buffer and try again */ so extend the output buffer and try again */
while ((err == Z_OK || err == Z_BUF_ERROR) && self->zst.avail_out == 0) { while ((err == Z_OK || err == Z_BUF_ERROR) && self->zst.avail_out == 0) {
if (_PyBytes_Resize(&retval, length << 1) < 0) { if (length <= (UINT_MAX >> 1))
new_length = length << 1;
else
new_length = UINT_MAX;
if (_PyBytes_Resize(&retval, new_length) < 0) {
Py_CLEAR(retval); Py_CLEAR(retval);
goto error; goto error;
} }
self->zst.next_out = (Byte *)PyBytes_AS_STRING(retval) + length; self->zst.next_out = (Byte *)PyBytes_AS_STRING(retval) + length;
self->zst.avail_out = length; self->zst.avail_out = length;
length = length << 1; length = new_length;
Py_BEGIN_ALLOW_THREADS Py_BEGIN_ALLOW_THREADS
err = inflate(&(self->zst), Z_FINISH); err = inflate(&(self->zst), Z_FINISH);
@ -1168,7 +1238,7 @@ PyZlib_adler32(PyObject *self, PyObject *args)
Py_BEGIN_ALLOW_THREADS Py_BEGIN_ALLOW_THREADS
/* Avoid truncation of length for very large buffers. adler32() takes /* Avoid truncation of length for very large buffers. adler32() takes
length as an unsigned int, which may be narrower than Py_ssize_t. */ length as an unsigned int, which may be narrower than Py_ssize_t. */
while (len > (size_t) UINT_MAX) { while ((size_t)len > UINT_MAX) {
adler32val = adler32(adler32val, buf, UINT_MAX); adler32val = adler32(adler32val, buf, UINT_MAX);
buf += (size_t) UINT_MAX; buf += (size_t) UINT_MAX;
len -= (size_t) UINT_MAX; len -= (size_t) UINT_MAX;
@ -1206,7 +1276,7 @@ PyZlib_crc32(PyObject *self, PyObject *args)
Py_BEGIN_ALLOW_THREADS Py_BEGIN_ALLOW_THREADS
/* Avoid truncation of length for very large buffers. crc32() takes /* Avoid truncation of length for very large buffers. crc32() takes
length as an unsigned int, which may be narrower than Py_ssize_t. */ length as an unsigned int, which may be narrower than Py_ssize_t. */
while (len > (size_t) UINT_MAX) { while ((size_t)len > UINT_MAX) {
crc32val = crc32(crc32val, buf, UINT_MAX); crc32val = crc32(crc32val, buf, UINT_MAX);
buf += (size_t) UINT_MAX; buf += (size_t) UINT_MAX;
len -= (size_t) UINT_MAX; len -= (size_t) UINT_MAX;