mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-31 05:58:33 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 7

Correction for issue1134: all source files with a coding spec, except latin-1

Browse source 
and utf-8, crashed when parsing a multiline string, or a line longer that 512
columns.
This commit is contained in:
Amaury Forgeot d'Arc 2007-11-15 23:19:43 +00:00
parent c05f42a8a7
commit 65f9aced6e
2 changed files with 50 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

22
Lib/test/test_coding.py
View file

@ -1,6 +1,6 @@
import test.test_support, unittest import test.test_support, unittest
import os import os, sys
class CodingTest(unittest.TestCase): class CodingTest(unittest.TestCase):
def test_bad_coding(self): def test_bad_coding(self):
@ -26,6 +26,26 @@ class CodingTest(unittest.TestCase):
exec('# coding: cp949\na = 5\n', d) exec('# coding: cp949\na = 5\n', d)
self.assertEqual(d['a'], 5) self.assertEqual(d['a'], 5)
def test_file_parse(self):
# issue1134: all encodings outside latin-1 and utf-8 fail on
# multiline strings and long lines (>512 columns)
sys.path.insert(0, ".")
filename = test.test_support.TESTFN+".py"
f = open(filename, "w")
try:
f.write("# -*- coding: cp1252 -*-\n")
f.write("'''A short string\n")
f.write("'''\n")
f.write("'A very long string %s'\n" % ("X" * 1000))
f.close()
__import__(test.test_support.TESTFN)
finally:
f.close()
os.remove(test.test_support.TESTFN+".py")
os.remove(test.test_support.TESTFN+".pyc")
sys.path.pop(0)
def test_main(): def test_main():
test.test_support.run_unittest(CodingTest) test.test_support.run_unittest(CodingTest)

43
Parser/tokenizer.c
View file

@ -369,46 +369,61 @@ check_bom(int get_char(struct tok_state *),
static char * static char *
fp_readl(char *s, int size, struct tok_state *tok) fp_readl(char *s, int size, struct tok_state *tok)
{ {
PyObject* bufobj = tok->decoding_buffer; PyObject* bufobj;
const char *buf; const char *buf;
Py_ssize_t buflen; Py_ssize_t buflen;
int allocated = 0;
/* Ask for one less byte so we can terminate it */ /* Ask for one less byte so we can terminate it */
assert(size > 0); assert(size > 0);
size--; size--;
if (bufobj == NULL) { if (tok->decoding_buffer) {
bufobj = tok->decoding_buffer;
Py_INCREF(bufobj);
}
else
{
bufobj = PyObject_CallObject(tok->decoding_readline, NULL); bufobj = PyObject_CallObject(tok->decoding_readline, NULL);
if (bufobj == NULL) if (bufobj == NULL)
goto error; goto error;
allocated = 1;
} }
buf = PyUnicode_AsStringAndSize(bufobj, &buflen); if (PyUnicode_CheckExact(bufobj))
if (buf == NULL) { {
goto error; buf = PyUnicode_AsStringAndSize(bufobj, &buflen);
if (buf == NULL) {
goto error;
}
} }
else
{
buf = PyBytes_AsString(bufobj);
if (buf == NULL) {
goto error;
}
buflen = PyBytes_GET_SIZE(bufobj);
}
Py_XDECREF(tok->decoding_buffer);
if (buflen > size) { if (buflen > size) {
Py_XDECREF(tok->decoding_buffer); /* Too many chars, the rest goes into tok->decoding_buffer */
tok->decoding_buffer = PyBytes_FromStringAndSize(buf+size, tok->decoding_buffer = PyBytes_FromStringAndSize(buf+size,
buflen-size); buflen-size);
if (tok->decoding_buffer == NULL) if (tok->decoding_buffer == NULL)
goto error; goto error;
buflen = size; buflen = size;
} }
else
tok->decoding_buffer = NULL;
memcpy(s, buf, buflen); memcpy(s, buf, buflen);
s[buflen] = '\0'; s[buflen] = '\0';
if (buflen == 0) /* EOF */ if (buflen == 0) /* EOF */
s = NULL; s = NULL;
if (allocated) { Py_DECREF(bufobj);
Py_DECREF(bufobj);
}
return s; return s;
error: error:
if (allocated) { Py_XDECREF(bufobj);
Py_XDECREF(bufobj);
}
return error_ret(tok); return error_ret(tok);
} }