mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-10-09 16:34:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 17

[3.6] bpo-30070: Fixed leaks and crashes in errors handling in the parser module. (GH-1131). (#1184)

Browse source 
(cherry picked from commit a79f4c2195)
This commit is contained in:
Serhiy Storchaka 2017-04-19 23:22:19 +03:00 • committed by GitHub
parent 680fea4067
commit 39dedb6e1a
3 changed files with 163 additions and 53 deletions
Download patch file Download diff file Expand all files Collapse all files

81
Lib/test/test_parser.py
View file

@ -1,4 +1,6 @@
import copy
import parser import parser
import pickle
import unittest import unittest
import operator import operator
import struct import struct
@ -424,6 +426,52 @@ class IllegalSyntaxTestCase(unittest.TestCase):
# not even remotely valid: # not even remotely valid:
self.check_bad_tree((1, 2, 3), "<junk>") self.check_bad_tree((1, 2, 3), "<junk>")
def test_illegal_terminal(self):
tree = \
(257,
(269,
(270,
(271,
(277,
(1,))),
(4, ''))),
(4, ''),
(0, ''))
self.check_bad_tree(tree, "too small items in terminal node")
tree = \
(257,
(269,
(270,
(271,
(277,
(1, b'pass'))),
(4, ''))),
(4, ''),
(0, ''))
self.check_bad_tree(tree, "non-string second item in terminal node")
tree = \
(257,
(269,
(270,
(271,
(277,
(1, 'pass', '0', 0))),
(4, ''))),
(4, ''),
(0, ''))
self.check_bad_tree(tree, "non-integer third item in terminal node")
tree = \
(257,
(269,
(270,
(271,
(277,
(1, 'pass', 0, 0))),
(4, ''))),
(4, ''),
(0, ''))
self.check_bad_tree(tree, "too many items in terminal node")
def test_illegal_yield_1(self): def test_illegal_yield_1(self):
# Illegal yield statement: def f(): return 1; yield 1 # Illegal yield statement: def f(): return 1; yield 1
tree = \ tree = \
@ -628,6 +676,24 @@ class IllegalSyntaxTestCase(unittest.TestCase):
(4, ''), (0, '')) (4, ''), (0, ''))
self.check_bad_tree(tree, "from import fred") self.check_bad_tree(tree, "from import fred")
def test_illegal_encoding(self):
# Illegal encoding declaration
tree = \
(339,
(257, (0, '')))
self.check_bad_tree(tree, "missed encoding")
tree = \
(339,
(257, (0, '')),
b'iso-8859-1')
self.check_bad_tree(tree, "non-string encoding")
tree = \
(339,
(257, (0, '')),
'\udcff')
with self.assertRaises(UnicodeEncodeError):
parser.sequence2st(tree)
class CompileTestCase(unittest.TestCase): class CompileTestCase(unittest.TestCase):
@ -772,6 +838,21 @@ class STObjectTestCase(unittest.TestCase):
self.assertRaises(TypeError, operator.lt, st1, 1815) self.assertRaises(TypeError, operator.lt, st1, 1815)
self.assertRaises(TypeError, operator.gt, b'waterloo', st2) self.assertRaises(TypeError, operator.gt, b'waterloo', st2)
def test_copy_pickle(self):
sts = [
parser.expr('2 + 3'),
parser.suite('x = 2; y = x + 3'),
parser.expr('list(x**3 for x in range(20))')
]
for st in sts:
st_copy = copy.copy(st)
self.assertEqual(st_copy.totuple(), st.totuple())
st_copy = copy.deepcopy(st)
self.assertEqual(st_copy.totuple(), st.totuple())
for proto in range(pickle.HIGHEST_PROTOCOL+1):
st_copy = pickle.loads(pickle.dumps(st, proto))
self.assertEqual(st_copy.totuple(), st.totuple())
check_sizeof = support.check_sizeof check_sizeof = support.check_sizeof
@support.cpython_only @support.cpython_only

2
Misc/NEWS
View file

@ -32,6 +32,8 @@ Core and Builtins
Library Library
------- -------
- bpo-30070: Fixed leaks and crashes in errors handling in the parser module.
- bpo-30061: Fixed crashes in IOBase methods __next__() and readlines() when - bpo-30061: Fixed crashes in IOBase methods __next__() and readlines() when
readline() or __next__() respectively return non-sizeable object. readline() or __next__() respectively return non-sizeable object.
Fixed possible other errors caused by not checking results of PyObject_Size(), Fixed possible other errors caused by not checking results of PyObject_Size(),

101
Modules/parsermodule.c
View file

@ -776,32 +776,35 @@ parser_tuple2st(PyST_Object *self, PyObject *args, PyObject *kw)
*/ */
tree = build_node_tree(tuple); tree = build_node_tree(tuple);
if (tree != 0) { if (tree != 0) {
node *validation_root = tree; node *validation_root = NULL;
int tree_type = 0; int tree_type = 0;
switch (TYPE(tree)) { switch (TYPE(tree)) {
case eval_input: case eval_input:
/* Might be an eval form. */ /* Might be an eval form. */
tree_type = PyST_EXPR; tree_type = PyST_EXPR;
validation_root = tree;
break; break;
case encoding_decl: case encoding_decl:
/* This looks like an encoding_decl so far. */ /* This looks like an encoding_decl so far. */
if (NCH(tree) != 1) if (NCH(tree) == 1) {
err_string("Error Parsing encoding_decl"); tree_type = PyST_SUITE;
validation_root = CHILD(tree, 0); validation_root = CHILD(tree, 0);
/* Fall through */ }
else {
err_string("Error Parsing encoding_decl");
}
break;
case file_input: case file_input:
/* This looks like an exec form so far. */ /* This looks like an exec form so far. */
tree_type = PyST_SUITE; tree_type = PyST_SUITE;
validation_root = tree;
break; break;
default: default:
/* This is a fragment, at best. */ /* This is a fragment, at best. */
PyNode_Free(tree);
err_string("parse tree does not use a valid start symbol"); err_string("parse tree does not use a valid start symbol");
return (0);
} }
if (validate_node(validation_root)) if (validation_root != NULL && validate_node(validation_root))
st = parser_newstobject(tree, tree_type); st = parser_newstobject(tree, tree_type);
else else
PyNode_Free(tree); PyNode_Free(tree);
@ -831,6 +834,9 @@ build_node_children(PyObject *tuple, node *root, int *line_num)
Py_ssize_t i; Py_ssize_t i;
int err; int err;
if (len < 0) {
return NULL;
}
for (i = 1; i < len; ++i) { for (i = 1; i < len; ++i) {
/* elem must always be a sequence, however simple */ /* elem must always be a sequence, however simple */
PyObject* elem = PySequence_GetItem(tuple, i); PyObject* elem = PySequence_GetItem(tuple, i);
@ -851,7 +857,7 @@ build_node_children(PyObject *tuple, node *root, int *line_num)
if (type == -1 && PyErr_Occurred()) { if (type == -1 && PyErr_Occurred()) {
Py_DECREF(temp); Py_DECREF(temp);
Py_DECREF(elem); Py_DECREF(elem);
return 0; return NULL;
} }
} }
Py_DECREF(temp); Py_DECREF(temp);
@ -863,7 +869,7 @@ build_node_children(PyObject *tuple, node *root, int *line_num)
PyErr_SetObject(parser_error, err); PyErr_SetObject(parser_error, err);
Py_XDECREF(err); Py_XDECREF(err);
Py_XDECREF(elem); Py_XDECREF(elem);
return (0); return NULL;
} }
if (ISTERMINAL(type)) { if (ISTERMINAL(type)) {
Py_ssize_t len = PyObject_Size(elem); Py_ssize_t len = PyObject_Size(elem);
@ -872,11 +878,14 @@ build_node_children(PyObject *tuple, node *root, int *line_num)
if ((len != 2) && (len != 3)) { if ((len != 2) && (len != 3)) {
err_string("terminal nodes must have 2 or 3 entries"); err_string("terminal nodes must have 2 or 3 entries");
return 0; Py_DECREF(elem);
return NULL;
} }
temp = PySequence_GetItem(elem, 1); temp = PySequence_GetItem(elem, 1);
if (temp == NULL) if (temp == NULL) {
return 0; Py_DECREF(elem);
return NULL;
}
if (!PyUnicode_Check(temp)) { if (!PyUnicode_Check(temp)) {
PyErr_Format(parser_error, PyErr_Format(parser_error,
"second item in terminal node must be a string," "second item in terminal node must be a string,"
@ -884,18 +893,22 @@ build_node_children(PyObject *tuple, node *root, int *line_num)
Py_TYPE(temp)->tp_name); Py_TYPE(temp)->tp_name);
Py_DECREF(temp); Py_DECREF(temp);
Py_DECREF(elem); Py_DECREF(elem);
return 0; return NULL;
} }
if (len == 3) { if (len == 3) {
PyObject *o = PySequence_GetItem(elem, 2); PyObject *o = PySequence_GetItem(elem, 2);
if (o != NULL) { if (o == NULL) {
Py_DECREF(temp);
Py_DECREF(elem);
return NULL;
}
if (PyLong_Check(o)) { if (PyLong_Check(o)) {
int num = _PyLong_AsInt(o); int num = _PyLong_AsInt(o);
if (num == -1 && PyErr_Occurred()) { if (num == -1 && PyErr_Occurred()) {
Py_DECREF(o); Py_DECREF(o);
Py_DECREF(temp); Py_DECREF(temp);
Py_DECREF(elem); Py_DECREF(elem);
return 0; return NULL;
} }
*line_num = num; *line_num = num;
} }
@ -907,23 +920,22 @@ build_node_children(PyObject *tuple, node *root, int *line_num)
Py_DECREF(o); Py_DECREF(o);
Py_DECREF(temp); Py_DECREF(temp);
Py_DECREF(elem); Py_DECREF(elem);
return 0; return NULL;
} }
Py_DECREF(o); Py_DECREF(o);
} }
}
temp_str = PyUnicode_AsUTF8AndSize(temp, &len); temp_str = PyUnicode_AsUTF8AndSize(temp, &len);
if (temp_str == NULL) { if (temp_str == NULL) {
Py_DECREF(temp); Py_DECREF(temp);
Py_XDECREF(elem); Py_DECREF(elem);
return 0; return NULL;
} }
strn = (char *)PyObject_MALLOC(len + 1); strn = (char *)PyObject_MALLOC(len + 1);
if (strn == NULL) { if (strn == NULL) {
Py_DECREF(temp); Py_DECREF(temp);
Py_XDECREF(elem); Py_DECREF(elem);
PyErr_NoMemory(); PyErr_NoMemory();
return 0; return NULL;
} }
(void) memcpy(strn, temp_str, len + 1); (void) memcpy(strn, temp_str, len + 1);
Py_DECREF(temp); Py_DECREF(temp);
@ -933,20 +945,21 @@ build_node_children(PyObject *tuple, node *root, int *line_num)
* It has to be one or the other; this is an error. * It has to be one or the other; this is an error.
* Raise an exception. * Raise an exception.
*/ */
PyObject *err = Py_BuildValue("os", elem, "unknown node type."); PyObject *err = Py_BuildValue("Os", elem, "unknown node type.");
PyErr_SetObject(parser_error, err); PyErr_SetObject(parser_error, err);
Py_XDECREF(err); Py_XDECREF(err);
Py_XDECREF(elem); Py_DECREF(elem);
return (0); return NULL;
} }
err = PyNode_AddChild(root, type, strn, *line_num, 0); err = PyNode_AddChild(root, type, strn, *line_num, 0);
if (err == E_NOMEM) { if (err == E_NOMEM) {
Py_XDECREF(elem); Py_DECREF(elem);
PyObject_FREE(strn); PyObject_FREE(strn);
return (node *) PyErr_NoMemory(); PyErr_NoMemory();
return NULL;
} }
if (err == E_OVERFLOW) { if (err == E_OVERFLOW) {
Py_XDECREF(elem); Py_DECREF(elem);
PyObject_FREE(strn); PyObject_FREE(strn);
PyErr_SetString(PyExc_ValueError, PyErr_SetString(PyExc_ValueError,
"unsupported number of child nodes"); "unsupported number of child nodes");
@ -957,14 +970,14 @@ build_node_children(PyObject *tuple, node *root, int *line_num)
node* new_child = CHILD(root, i - 1); node* new_child = CHILD(root, i - 1);
if (new_child != build_node_children(elem, new_child, line_num)) { if (new_child != build_node_children(elem, new_child, line_num)) {
Py_XDECREF(elem); Py_DECREF(elem);
return (0); return NULL;
} }
} }
else if (type == NEWLINE) { /* It's true: we increment the */ else if (type == NEWLINE) { /* It's true: we increment the */
++(*line_num); /* line number *after* the newline! */ ++(*line_num); /* line number *after* the newline! */
} }
Py_XDECREF(elem); Py_DECREF(elem);
} }
return root; return root;
} }
@ -999,11 +1012,24 @@ build_node_tree(PyObject *tuple)
if (num == encoding_decl) { if (num == encoding_decl) {
encoding = PySequence_GetItem(tuple, 2); encoding = PySequence_GetItem(tuple, 2);
if (encoding == NULL) {
PyErr_SetString(parser_error, "missed encoding");
return NULL;
}
if (!PyUnicode_Check(encoding)) {
PyErr_Format(parser_error,
"encoding must be a string, found %.200s",
Py_TYPE(encoding)->tp_name);
Py_DECREF(encoding);
return NULL;
}
/* tuple isn't borrowed anymore here, need to DECREF */ /* tuple isn't borrowed anymore here, need to DECREF */
tuple = PySequence_GetSlice(tuple, 0, 2); tuple = PySequence_GetSlice(tuple, 0, 2);
if (tuple == NULL) if (tuple == NULL) {
Py_DECREF(encoding);
return NULL; return NULL;
} }
}
res = PyNode_New(num); res = PyNode_New(num);
if (res != NULL) { if (res != NULL) {
if (res != build_node_children(tuple, res, &line_num)) { if (res != build_node_children(tuple, res, &line_num)) {
@ -1015,31 +1041,33 @@ build_node_tree(PyObject *tuple)
const char *temp; const char *temp;
temp = PyUnicode_AsUTF8AndSize(encoding, &len); temp = PyUnicode_AsUTF8AndSize(encoding, &len);
if (temp == NULL) { if (temp == NULL) {
Py_DECREF(res); PyNode_Free(res);
Py_DECREF(encoding); Py_DECREF(encoding);
Py_DECREF(tuple); Py_DECREF(tuple);
return NULL; return NULL;
} }
res->n_str = (char *)PyObject_MALLOC(len + 1); res->n_str = (char *)PyObject_MALLOC(len + 1);
if (res->n_str == NULL) { if (res->n_str == NULL) {
Py_DECREF(res); PyNode_Free(res);
Py_DECREF(encoding); Py_DECREF(encoding);
Py_DECREF(tuple); Py_DECREF(tuple);
PyErr_NoMemory(); PyErr_NoMemory();
return NULL; return NULL;
} }
(void) memcpy(res->n_str, temp, len + 1); (void) memcpy(res->n_str, temp, len + 1);
}
}
if (encoding != NULL) {
Py_DECREF(encoding); Py_DECREF(encoding);
Py_DECREF(tuple); Py_DECREF(tuple);
} }
} }
}
else { else {
/* The tuple is illegal -- if the number is neither TERMINAL nor /* The tuple is illegal -- if the number is neither TERMINAL nor
* NONTERMINAL, we can't use it. Not sure the implementation * NONTERMINAL, we can't use it. Not sure the implementation
* allows this condition, but the API doesn't preclude it. * allows this condition, but the API doesn't preclude it.
*/ */
PyObject *err = Py_BuildValue("os", tuple, PyObject *err = Py_BuildValue("Os", tuple,
"Illegal component tuple."); "Illegal component tuple.");
PyErr_SetObject(parser_error, err); PyErr_SetObject(parser_error, err);
Py_XDECREF(err); Py_XDECREF(err);
@ -1074,7 +1102,6 @@ parser__pickler(PyObject *self, PyObject *args)
result = Py_BuildValue("O(O)", pickle_constructor, tuple); result = Py_BuildValue("O(O)", pickle_constructor, tuple);
Py_DECREF(tuple); Py_DECREF(tuple);
} }
Py_DECREF(empty_dict);
Py_DECREF(newargs); Py_DECREF(newargs);
} }
finally: finally: