mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-07-24 11:44:31 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

gh-105042: Disable unmatched parens syntax error in python tokenize (#105061)

Browse source
This commit is contained in:
Lysandros Nikolaou 2023-05-30 23:52:52 +02:00 • committed by GitHub
parent 9216e69a87
commit 70f315c2d6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 49 additions and 34 deletions
Download patch file Download diff file Expand all files Collapse all files

5
Lib/test/inspect_fodder.py
View file

@ -113,3 +113,8 @@ class WhichComments:
# after asyncf - line 113 # after asyncf - line 113
# end of WhichComments - line 114 # end of WhichComments - line 114
# after WhichComments - line 115 # after WhichComments - line 115
# Test that getsource works on a line that includes
# a closing parenthesis with the opening paren being in another line
(
); after_closing = lambda: 1

4
Lib/test/test_inspect.py
View file

@ -557,7 +557,8 @@ class TestRetrievingSourceCode(GetSourceBase):
def test_getfunctions(self): def test_getfunctions(self):
functions = inspect.getmembers(mod, inspect.isfunction) functions = inspect.getmembers(mod, inspect.isfunction)
self.assertEqual(functions, [('eggs', mod.eggs), self.assertEqual(functions, [('after_closing', mod.after_closing),
('eggs', mod.eggs),
('lobbest', mod.lobbest), ('lobbest', mod.lobbest),
('spam', mod.spam)]) ('spam', mod.spam)])
@ -641,6 +642,7 @@ class TestRetrievingSourceCode(GetSourceBase):
self.assertSourceEqual(git.abuse, 29, 39) self.assertSourceEqual(git.abuse, 29, 39)
self.assertSourceEqual(mod.StupidGit, 21, 51) self.assertSourceEqual(mod.StupidGit, 21, 51)
self.assertSourceEqual(mod.lobbest, 75, 76) self.assertSourceEqual(mod.lobbest, 75, 76)
self.assertSourceEqual(mod.after_closing, 120, 120)
def test_getsourcefile(self): def test_getsourcefile(self):
self.assertEqual(normcase(inspect.getsourcefile(mod.spam)), modfile) self.assertEqual(normcase(inspect.getsourcefile(mod.spam)), modfile)

7
Lib/test/test_tokenize.py
View file

@ -1119,6 +1119,13 @@ async def f():
NEWLINE '\\n' (4, 1) (4, 2) NEWLINE '\\n' (4, 1) (4, 2)
""") """)
def test_closing_parenthesis_from_different_line(self):
self.check_tokenize("); x", """\
OP ')' (1, 0) (1, 1)
OP ';' (1, 1) (1, 2)
NAME 'x' (1, 3) (1, 4)
""")
class GenerateTokensTest(TokenizeTest): class GenerateTokensTest(TokenizeTest):
def check_tokenize(self, s, expected): def check_tokenize(self, s, expected):
# Format the tokens in s in a table format. # Format the tokens in s in a table format.

65
Parser/tokenizer.c
View file

@ -2626,41 +2626,42 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t
case ')': case ')':
case ']': case ']':
case '}': case '}':
if (!tok->level) { if (INSIDE_FSTRING(tok) && !current_tok->curly_bracket_depth && c == '}') {
if (INSIDE_FSTRING(tok) && !current_tok->curly_bracket_depth && c == '}') { return MAKE_TOKEN(syntaxerror(tok, "f-string: single '}' is not allowed"));
return MAKE_TOKEN(syntaxerror(tok, "f-string: single '}' is not allowed")); }
} if (!tok->tok_extra_tokens && !tok->level) {
return MAKE_TOKEN(syntaxerror(tok, "unmatched '%c'", c)); return MAKE_TOKEN(syntaxerror(tok, "unmatched '%c'", c));
} }
tok->level--; if (tok->level > 0) {
int opening = tok->parenstack[tok->level]; tok->level--;
if (!((opening == '(' && c == ')') || int opening = tok->parenstack[tok->level];
(opening == '[' && c == ']') || if (!tok->tok_extra_tokens && !((opening == '(' && c == ')') ||
(opening == '{' && c == '}'))) (opening == '[' && c == ']') ||
{ (opening == '{' && c == '}'))) {
/* If the opening bracket belongs to an f-string's expression /* If the opening bracket belongs to an f-string's expression
part (e.g. f"{)}") and the closing bracket is an arbitrary part (e.g. f"{)}") and the closing bracket is an arbitrary
nested expression, then instead of matching a different nested expression, then instead of matching a different
syntactical construct with it; we'll throw an unmatched syntactical construct with it; we'll throw an unmatched
parentheses error. */ parentheses error. */
if (INSIDE_FSTRING(tok) && opening == '{') { if (INSIDE_FSTRING(tok) && opening == '{') {
assert(current_tok->curly_bracket_depth >= 0); assert(current_tok->curly_bracket_depth >= 0);
int previous_bracket = current_tok->curly_bracket_depth - 1; int previous_bracket = current_tok->curly_bracket_depth - 1;
if (previous_bracket == current_tok->curly_bracket_expr_start_depth) { if (previous_bracket == current_tok->curly_bracket_expr_start_depth) {
return MAKE_TOKEN(syntaxerror(tok, "f-string: unmatched '%c'", c)); return MAKE_TOKEN(syntaxerror(tok, "f-string: unmatched '%c'", c));
}
}
if (tok->parenlinenostack[tok->level] != tok->lineno) {
return MAKE_TOKEN(syntaxerror(tok,
"closing parenthesis '%c' does not match "
"opening parenthesis '%c' on line %d",
c, opening, tok->parenlinenostack[tok->level]));
}
else {
return MAKE_TOKEN(syntaxerror(tok,
"closing parenthesis '%c' does not match "
"opening parenthesis '%c'",
c, opening));
} }
}
if (tok->parenlinenostack[tok->level] != tok->lineno) {
return MAKE_TOKEN(syntaxerror(tok,
"closing parenthesis '%c' does not match "
"opening parenthesis '%c' on line %d",
c, opening, tok->parenlinenostack[tok->level]));
}
else {
return MAKE_TOKEN(syntaxerror(tok,
"closing parenthesis '%c' does not match "
"opening parenthesis '%c'",
c, opening));
} }
} }

2
Python/Python-tokenize.c
View file

@ -84,7 +84,7 @@ _tokenizer_error(struct tok_state *tok)
msg = "invalid token"; msg = "invalid token";
break; break;
case E_EOF: case E_EOF:
if (tok->level) { if (tok->level > 0) {
PyErr_Format(PyExc_SyntaxError, PyErr_Format(PyExc_SyntaxError,
"parenthesis '%c' was never closed", "parenthesis '%c' was never closed",
tok->parenstack[tok->level-1]); tok->parenstack[tok->level-1]);