mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-09-27 02:39:58 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 13

Issue #5308: Raise ValueError when marshalling too large object (a sequence

Browse source 
with size >= 2**31), instead of producing illegal marshal data.
This commit is contained in:
Serhiy Storchaka 2013-02-13 12:07:43 +02:00
parent 7d360038f5
commit 34fe1b7a3d
4 changed files with 116 additions and 63 deletions
Download patch file Download diff file Expand all files Collapse all files

51
Lib/test/test_marshal.py
View file

@ -269,6 +269,53 @@ class BugsTestCase(unittest.TestCase):
invalid_string = 'l\x02\x00\x00\x00\x00\x00\x00\x00' invalid_string = 'l\x02\x00\x00\x00\x00\x00\x00\x00'
self.assertRaises(ValueError, marshal.loads, invalid_string) self.assertRaises(ValueError, marshal.loads, invalid_string)
LARGE_SIZE = 2**31
character_size = 4 if sys.maxunicode > 0xFFFF else 2
pointer_size = 8 if sys.maxsize > 0xFFFFFFFF else 4
@unittest.skipIf(LARGE_SIZE > sys.maxsize, "test cannot run on 32-bit systems")
class LargeValuesTestCase(unittest.TestCase):
def check_unmarshallable(self, data):
f = open(test_support.TESTFN, 'wb')
self.addCleanup(test_support.unlink, test_support.TESTFN)
with f:
self.assertRaises(ValueError, marshal.dump, data, f)
@test_support.precisionbigmemtest(size=LARGE_SIZE, memuse=1, dry_run=False)
def test_bytes(self, size):
self.check_unmarshallable(b'x' * size)
@test_support.precisionbigmemtest(size=LARGE_SIZE,
memuse=character_size, dry_run=False)
def test_str(self, size):
self.check_unmarshallable('x' * size)
@test_support.precisionbigmemtest(size=LARGE_SIZE,
memuse=pointer_size, dry_run=False)
def test_tuple(self, size):
self.check_unmarshallable((None,) * size)
@test_support.precisionbigmemtest(size=LARGE_SIZE,
memuse=pointer_size, dry_run=False)
def test_list(self, size):
self.check_unmarshallable([None] * size)
@test_support.precisionbigmemtest(size=LARGE_SIZE,
memuse=pointer_size*12 + sys.getsizeof(LARGE_SIZE-1),
dry_run=False)
def test_set(self, size):
self.check_unmarshallable(set(range(size)))
@test_support.precisionbigmemtest(size=LARGE_SIZE,
memuse=pointer_size*12 + sys.getsizeof(LARGE_SIZE-1),
dry_run=False)
def test_frozenset(self, size):
self.check_unmarshallable(frozenset(range(size)))
@test_support.precisionbigmemtest(size=LARGE_SIZE, memuse=1, dry_run=False)
def test_bytearray(self, size):
self.check_unmarshallable(bytearray(size))
def test_main(): def test_main():
test_support.run_unittest(IntTestCase, test_support.run_unittest(IntTestCase,
@ -277,7 +324,9 @@ def test_main():
CodeTestCase, CodeTestCase,
ContainerTestCase, ContainerTestCase,
ExceptionTestCase, ExceptionTestCase,
BugsTestCase) BugsTestCase,
LargeValuesTestCase,
)
if __name__ == "__main__": if __name__ == "__main__":
test_main() test_main()

13
Lib/test/test_support.py
View file

@ -1062,7 +1062,7 @@ def bigmemtest(minsize, memuse, overhead=5*_1M):
return wrapper return wrapper
return decorator return decorator
def precisionbigmemtest(size, memuse, overhead=5*_1M): def precisionbigmemtest(size, memuse, overhead=5*_1M, dry_run=True):
def decorator(f): def decorator(f):
def wrapper(self): def wrapper(self):
if not real_max_memuse: if not real_max_memuse:
@ -1070,11 +1070,12 @@ def precisionbigmemtest(size, memuse, overhead=5*_1M):
else: else:
maxsize = size maxsize = size
if real_max_memuse and real_max_memuse < maxsize * memuse: if ((real_max_memuse or not dry_run)
if verbose: and real_max_memuse < maxsize * memuse):
sys.stderr.write("Skipping %s because of memory " if verbose:
"constraint\n" % (f.__name__,)) sys.stderr.write("Skipping %s because of memory "
return "constraint\n" % (f.__name__,))
return
return f(self, maxsize) return f(self, maxsize)
wrapper.size = size wrapper.size = size

3
Misc/NEWS
View file

@ -9,6 +9,9 @@ What's New in Python 2.7.4
Core and Builtins Core and Builtins
----------------- -----------------
- Issue #5308: Raise ValueError when marshalling too large object (a sequence
with size >= 2**31), instead of producing illegal marshal data.
- Issue #17043: The unicode-internal decoder no longer read past the end of - Issue #17043: The unicode-internal decoder no longer read past the end of
input buffer. input buffer.

112
Python/marshal.c
View file

@ -88,7 +88,7 @@ w_more(int c, WFILE *p)
} }
static void static void
w_string(char *s, int n, WFILE *p) w_string(char *s, Py_ssize_t n, WFILE *p)
{ {
if (p->fp != NULL) { if (p->fp != NULL) {
fwrite(s, 1, n, p->fp); fwrite(s, 1, n, p->fp);
@ -126,6 +126,21 @@ w_long64(long x, WFILE *p)
} }
#endif #endif
#define SIZE32_MAX 0x7FFFFFFF
#if SIZEOF_SIZE_T > 4
# define W_SIZE(n, p) do { \
if ((n) > SIZE32_MAX) { \
(p)->depth--; \
(p)->error = WFERR_UNMARSHALLABLE; \
return; \
} \
w_long((long)(n), p); \
} while(0)
#else
# define W_SIZE w_long
#endif
/* We assume that Python longs are stored internally in base some power of /* We assume that Python longs are stored internally in base some power of
2**15; for the sake of portability we'll always read and write them in base 2**15; for the sake of portability we'll always read and write them in base
exactly 2**15. */ exactly 2**15. */
@ -159,6 +174,11 @@ w_PyLong(const PyLongObject *ob, WFILE *p)
d >>= PyLong_MARSHAL_SHIFT; d >>= PyLong_MARSHAL_SHIFT;
l++; l++;
} while (d != 0); } while (d != 0);
if (l > SIZE32_MAX) {
p->depth--;
p->error = WFERR_UNMARSHALLABLE;
return;
}
w_long((long)(Py_SIZE(ob) > 0 ? l : -l), p); w_long((long)(Py_SIZE(ob) > 0 ? l : -l), p);
for (i=0; i < n-1; i++) { for (i=0; i < n-1; i++) {
@ -244,7 +264,7 @@ w_object(PyObject *v, WFILE *p)
n = strlen(buf); n = strlen(buf);
w_byte(TYPE_FLOAT, p); w_byte(TYPE_FLOAT, p);
w_byte((int)n, p); w_byte((int)n, p);
w_string(buf, (int)n, p); w_string(buf, n, p);
PyMem_Free(buf); PyMem_Free(buf);
} }
} }
@ -277,7 +297,7 @@ w_object(PyObject *v, WFILE *p)
} }
n = strlen(buf); n = strlen(buf);
w_byte((int)n, p); w_byte((int)n, p);
w_string(buf, (int)n, p); w_string(buf, n, p);
PyMem_Free(buf); PyMem_Free(buf);
buf = PyOS_double_to_string(PyComplex_ImagAsDouble(v), buf = PyOS_double_to_string(PyComplex_ImagAsDouble(v),
'g', 17, 0, NULL); 'g', 17, 0, NULL);
@ -287,7 +307,7 @@ w_object(PyObject *v, WFILE *p)
} }
n = strlen(buf); n = strlen(buf);
w_byte((int)n, p); w_byte((int)n, p);
w_string(buf, (int)n, p); w_string(buf, n, p);
PyMem_Free(buf); PyMem_Free(buf);
} }
} }
@ -319,14 +339,8 @@ w_object(PyObject *v, WFILE *p)
w_byte(TYPE_STRING, p); w_byte(TYPE_STRING, p);
} }
n = PyString_GET_SIZE(v); n = PyString_GET_SIZE(v);
if (n > INT_MAX) { W_SIZE(n, p);
/* huge strings are not supported */ w_string(PyString_AS_STRING(v), n, p);
p->depth--;
p->error = WFERR_UNMARSHALLABLE;
return;
}
w_long((long)n, p);
w_string(PyString_AS_STRING(v), (int)n, p);
} }
#ifdef Py_USING_UNICODE #ifdef Py_USING_UNICODE
else if (PyUnicode_CheckExact(v)) { else if (PyUnicode_CheckExact(v)) {
@ -339,20 +353,15 @@ w_object(PyObject *v, WFILE *p)
} }
w_byte(TYPE_UNICODE, p); w_byte(TYPE_UNICODE, p);
n = PyString_GET_SIZE(utf8); n = PyString_GET_SIZE(utf8);
if (n > INT_MAX) { W_SIZE(n, p);
p->depth--; w_string(PyString_AS_STRING(utf8), n, p);
p->error = WFERR_UNMARSHALLABLE;
return;
}
w_long((long)n, p);
w_string(PyString_AS_STRING(utf8), (int)n, p);
Py_DECREF(utf8); Py_DECREF(utf8);
} }
#endif #endif
else if (PyTuple_CheckExact(v)) { else if (PyTuple_CheckExact(v)) {
w_byte(TYPE_TUPLE, p); w_byte(TYPE_TUPLE, p);
n = PyTuple_Size(v); n = PyTuple_Size(v);
w_long((long)n, p); W_SIZE(n, p);
for (i = 0; i < n; i++) { for (i = 0; i < n; i++) {
w_object(PyTuple_GET_ITEM(v, i), p); w_object(PyTuple_GET_ITEM(v, i), p);
} }
@ -360,7 +369,7 @@ w_object(PyObject *v, WFILE *p)
else if (PyList_CheckExact(v)) { else if (PyList_CheckExact(v)) {
w_byte(TYPE_LIST, p); w_byte(TYPE_LIST, p);
n = PyList_GET_SIZE(v); n = PyList_GET_SIZE(v);
w_long((long)n, p); W_SIZE(n, p);
for (i = 0; i < n; i++) { for (i = 0; i < n; i++) {
w_object(PyList_GET_ITEM(v, i), p); w_object(PyList_GET_ITEM(v, i), p);
} }
@ -390,7 +399,7 @@ w_object(PyObject *v, WFILE *p)
p->error = WFERR_UNMARSHALLABLE; p->error = WFERR_UNMARSHALLABLE;
return; return;
} }
w_long((long)n, p); W_SIZE(n, p);
it = PyObject_GetIter(v); it = PyObject_GetIter(v);
if (it == NULL) { if (it == NULL) {
p->depth--; p->depth--;
@ -432,13 +441,8 @@ w_object(PyObject *v, WFILE *p)
PyBufferProcs *pb = v->ob_type->tp_as_buffer; PyBufferProcs *pb = v->ob_type->tp_as_buffer;
w_byte(TYPE_STRING, p); w_byte(TYPE_STRING, p);
n = (*pb->bf_getreadbuffer)(v, 0, (void **)&s); n = (*pb->bf_getreadbuffer)(v, 0, (void **)&s);
if (n > INT_MAX) { W_SIZE(n, p);
p->depth--; w_string(s, n, p);
p->error = WFERR_UNMARSHALLABLE;
return;
}
w_long((long)n, p);
w_string(s, (int)n, p);
} }
else { else {
w_byte(TYPE_UNKNOWN, p); w_byte(TYPE_UNKNOWN, p);
@ -480,14 +484,14 @@ typedef WFILE RFILE; /* Same struct with different invariants */
#define r_byte(p) ((p)->fp ? getc((p)->fp) : rs_byte(p)) #define r_byte(p) ((p)->fp ? getc((p)->fp) : rs_byte(p))
static int static Py_ssize_t
r_string(char *s, int n, RFILE *p) r_string(char *s, Py_ssize_t n, RFILE *p)
{ {
if (p->fp != NULL) if (p->fp != NULL)
/* The result fits into int because it must be <=n. */ /* The result fits into int because it must be <=n. */
return (int)fread(s, 1, n, p->fp); return fread(s, 1, n, p->fp);
if (p->end - p->ptr < n) if (p->end - p->ptr < n)
n = (int)(p->end - p->ptr); n = p->end - p->ptr;
memcpy(s, p->ptr, n); memcpy(s, p->ptr, n);
p->ptr += n; p->ptr += n;
return n; return n;
@ -563,14 +567,14 @@ static PyObject *
r_PyLong(RFILE *p) r_PyLong(RFILE *p)
{ {
PyLongObject *ob; PyLongObject *ob;
int size, i, j, md, shorts_in_top_digit; long n, size, i;
long n; int j, md, shorts_in_top_digit;
digit d; digit d;
n = r_long(p); n = r_long(p);
if (n == 0) if (n == 0)
return (PyObject *)_PyLong_New(0); return (PyObject *)_PyLong_New(0);
if (n < -INT_MAX || n > INT_MAX) { if (n < -SIZE32_MAX || n > SIZE32_MAX) {
PyErr_SetString(PyExc_ValueError, PyErr_SetString(PyExc_ValueError,
"bad marshal data (long size out of range)"); "bad marshal data (long size out of range)");
return NULL; return NULL;
@ -691,7 +695,7 @@ r_object(RFILE *p)
char buf[256]; char buf[256];
double dx; double dx;
n = r_byte(p); n = r_byte(p);
if (n == EOF || r_string(buf, (int)n, p) != n) { if (n == EOF || r_string(buf, n, p) != n) {
PyErr_SetString(PyExc_EOFError, PyErr_SetString(PyExc_EOFError,
"EOF read where object expected"); "EOF read where object expected");
retval = NULL; retval = NULL;
@ -732,7 +736,7 @@ r_object(RFILE *p)
char buf[256]; char buf[256];
Py_complex c; Py_complex c;
n = r_byte(p); n = r_byte(p);
if (n == EOF || r_string(buf, (int)n, p) != n) { if (n == EOF || r_string(buf, n, p) != n) {
PyErr_SetString(PyExc_EOFError, PyErr_SetString(PyExc_EOFError,
"EOF read where object expected"); "EOF read where object expected");
retval = NULL; retval = NULL;
@ -745,7 +749,7 @@ r_object(RFILE *p)
break; break;
} }
n = r_byte(p); n = r_byte(p);
if (n == EOF || r_string(buf, (int)n, p) != n) { if (n == EOF || r_string(buf, n, p) != n) {
PyErr_SetString(PyExc_EOFError, PyErr_SetString(PyExc_EOFError,
"EOF read where object expected"); "EOF read where object expected");
retval = NULL; retval = NULL;
@ -795,7 +799,7 @@ r_object(RFILE *p)
case TYPE_INTERNED: case TYPE_INTERNED:
case TYPE_STRING: case TYPE_STRING:
n = r_long(p); n = r_long(p);
if (n < 0 || n > INT_MAX) { if (n < 0 || n > SIZE32_MAX) {
PyErr_SetString(PyExc_ValueError, "bad marshal data (string size out of range)"); PyErr_SetString(PyExc_ValueError, "bad marshal data (string size out of range)");
retval = NULL; retval = NULL;
break; break;
@ -805,7 +809,7 @@ r_object(RFILE *p)
retval = NULL; retval = NULL;
break; break;
} }
if (r_string(PyString_AS_STRING(v), (int)n, p) != n) { if (r_string(PyString_AS_STRING(v), n, p) != n) {
Py_DECREF(v); Py_DECREF(v);
PyErr_SetString(PyExc_EOFError, PyErr_SetString(PyExc_EOFError,
"EOF read where object expected"); "EOF read where object expected");
@ -840,7 +844,7 @@ r_object(RFILE *p)
char *buffer; char *buffer;
n = r_long(p); n = r_long(p);
if (n < 0 || n > INT_MAX) { if (n < 0 || n > SIZE32_MAX) {
PyErr_SetString(PyExc_ValueError, "bad marshal data (unicode size out of range)"); PyErr_SetString(PyExc_ValueError, "bad marshal data (unicode size out of range)");
retval = NULL; retval = NULL;
break; break;
@ -850,7 +854,7 @@ r_object(RFILE *p)
retval = PyErr_NoMemory(); retval = PyErr_NoMemory();
break; break;
} }
if (r_string(buffer, (int)n, p) != n) { if (r_string(buffer, n, p) != n) {
PyMem_DEL(buffer); PyMem_DEL(buffer);
PyErr_SetString(PyExc_EOFError, PyErr_SetString(PyExc_EOFError,
"EOF read where object expected"); "EOF read where object expected");
@ -866,12 +870,12 @@ r_object(RFILE *p)
case TYPE_TUPLE: case TYPE_TUPLE:
n = r_long(p); n = r_long(p);
if (n < 0 || n > INT_MAX) { if (n < 0 || n > SIZE32_MAX) {
PyErr_SetString(PyExc_ValueError, "bad marshal data (tuple size out of range)"); PyErr_SetString(PyExc_ValueError, "bad marshal data (tuple size out of range)");
retval = NULL; retval = NULL;
break; break;
} }
v = PyTuple_New((int)n); v = PyTuple_New(n);
if (v == NULL) { if (v == NULL) {
retval = NULL; retval = NULL;
break; break;
@ -886,19 +890,19 @@ r_object(RFILE *p)
v = NULL; v = NULL;
break; break;
} }
PyTuple_SET_ITEM(v, (int)i, v2); PyTuple_SET_ITEM(v, i, v2);
} }
retval = v; retval = v;
break; break;
case TYPE_LIST: case TYPE_LIST:
n = r_long(p); n = r_long(p);
if (n < 0 || n > INT_MAX) { if (n < 0 || n > SIZE32_MAX) {
PyErr_SetString(PyExc_ValueError, "bad marshal data (list size out of range)"); PyErr_SetString(PyExc_ValueError, "bad marshal data (list size out of range)");
retval = NULL; retval = NULL;
break; break;
} }
v = PyList_New((int)n); v = PyList_New(n);
if (v == NULL) { if (v == NULL) {
retval = NULL; retval = NULL;
break; break;
@ -913,7 +917,7 @@ r_object(RFILE *p)
v = NULL; v = NULL;
break; break;
} }
PyList_SET_ITEM(v, (int)i, v2); PyList_SET_ITEM(v, i, v2);
} }
retval = v; retval = v;
break; break;
@ -945,7 +949,7 @@ r_object(RFILE *p)
case TYPE_SET: case TYPE_SET:
case TYPE_FROZENSET: case TYPE_FROZENSET:
n = r_long(p); n = r_long(p);
if (n < 0 || n > INT_MAX) { if (n < 0 || n > SIZE32_MAX) {
PyErr_SetString(PyExc_ValueError, "bad marshal data (set size out of range)"); PyErr_SetString(PyExc_ValueError, "bad marshal data (set size out of range)");
retval = NULL; retval = NULL;
break; break;
@ -1134,12 +1138,8 @@ PyMarshal_ReadLastObjectFromFile(FILE *fp)
if (filesize > 0 && filesize <= REASONABLE_FILE_LIMIT) { if (filesize > 0 && filesize <= REASONABLE_FILE_LIMIT) {
char* pBuf = (char *)PyMem_MALLOC(filesize); char* pBuf = (char *)PyMem_MALLOC(filesize);
if (pBuf != NULL) { if (pBuf != NULL) {
PyObject* v; size_t n = fread(pBuf, 1, (size_t)filesize, fp);
size_t n; PyObject* v = PyMarshal_ReadObjectFromString(pBuf, n);
/* filesize must fit into an int, because it
is smaller than REASONABLE_FILE_LIMIT */
n = fread(pBuf, 1, (int)filesize, fp);
v = PyMarshal_ReadObjectFromString(pBuf, n);
PyMem_FREE(pBuf); PyMem_FREE(pBuf);
return v; return v;
} }